Organize Python tests to use grpcio-tools directly

Moves all tests into a separate package. This does not change existing
supported means of running tests (e.g. through run_tests.py).

.gitignore

@@ -13,6 +13,8 @@ python_build/
 htmlcov/
 dist/
 *.egg
+py27/
+py34/
 
 # Node installation output
 ^node_modules

setup.py

@@ -72,10 +72,6 @@ BUILD_WITH_CYTHON = os.environ.get('GRPC_PYTHON_BUILD_WITH_CYTHON', False)
 ENABLE_CYTHON_TRACING = os.environ.get(
     'GRPC_PYTHON_ENABLE_CYTHON_TRACING', False)
 
-# Environment variable to determine whether or not to include the test files in
-# the installation.
-INSTALL_TESTS = os.environ.get('GRPC_PYTHON_INSTALL_TESTS', False)
-
 CYTHON_EXTENSION_PACKAGE_NAMES = ()
 
 CYTHON_EXTENSION_MODULE_NAMES = ('grpc._cython.cygrpc',)
@@ -183,13 +179,10 @@ SETUP_REQUIRES = INSTALL_REQUIRES + (
 
 COMMAND_CLASS = {
     'doc': commands.SphinxDocumentation,
-    'build_proto_modules': commands.BuildProtoModules,
     'build_project_metadata': commands.BuildProjectMetadata,
     'build_py': commands.BuildPy,
     'build_ext': commands.BuildExt,
     'gather': commands.Gather,
-    'run_interop': commands.RunInterop,
-    'test_lite': commands.TestLite
 }
 
 # Ensure that package data is copied over before any commands have been run:
@@ -200,32 +193,6 @@ except OSError:
   pass
 shutil.copyfile('etc/roots.pem', os.path.join(credentials_dir, 'roots.pem'))
 
-TEST_PACKAGE_DATA = {
-    'tests.interop': [
-        'credentials/ca.pem',
-        'credentials/server1.key',
-        'credentials/server1.pem',
-    ],
-    'tests.protoc_plugin': [
-        'protoc_plugin_test.proto',
-    ],
-    'tests.unit': [
-        'credentials/ca.pem',
-        'credentials/server1.key',
-        'credentials/server1.pem',
-    ],
-}
-
-TESTS_REQUIRE = (
-    'oauth2client>=2.1.0',
-    'protobuf>=3.0.0a3',
-    'coverage>=4.0',
-) + INSTALL_REQUIRES
-
-TEST_SUITE = 'tests'
-TEST_LOADER = 'tests:Loader'
-TEST_RUNNER = 'tests:Runner'
-
 PACKAGE_DATA = {
     # Binaries that may or may not be present in the final installation, but are
     # mentioned here for completeness.
@@ -235,12 +202,7 @@ PACKAGE_DATA = {
         '_windows/grpc_c.64.python',
     ],
 }
-if INSTALL_TESTS:
+PACKAGES = setuptools.find_packages(PYTHON_STEM)
-  PACKAGE_DATA = dict(PACKAGE_DATA, **TEST_PACKAGE_DATA)
-  PACKAGES = setuptools.find_packages(PYTHON_STEM)
-else:
-  PACKAGES = setuptools.find_packages(
-      PYTHON_STEM, exclude=['tests', 'tests.*'])
 
 setuptools.setup(
   name='grpcio',
@@ -253,8 +215,4 @@ setuptools.setup(
   install_requires=INSTALL_REQUIRES,
   setup_requires=SETUP_REQUIRES,
   cmdclass=COMMAND_CLASS,
-  tests_require=TESTS_REQUIRE,
-  test_suite=TEST_SUITE,
-  test_loader=TEST_LOADER,
-  test_runner=TEST_RUNNER,
 )

src/python/grpcio/commands.py

@@ -134,75 +134,6 @@ class SphinxDocumentation(setuptools.Command):
     sphinx.main(['', os.path.join('doc', 'src'), os.path.join('doc', 'build')])
 
 
-class BuildProtoModules(setuptools.Command):
-  """Command to generate project *_pb2.py modules from proto files."""
-
-  description = 'build protobuf modules'
-  user_options = [
-    ('include=', None, 'path patterns to include in protobuf generation'),
-    ('exclude=', None, 'path patterns to exclude from protobuf generation')
-  ]
-
-  def initialize_options(self):
-    self.exclude = None
-    self.include = r'.*\.proto$'
-    self.protoc_command = None
-    self.grpc_python_plugin_command = None
-
-  def finalize_options(self):
-    self.protoc_command = distutils.spawn.find_executable('protoc')
-    self.grpc_python_plugin_command = distutils.spawn.find_executable(
-        'grpc_python_plugin')
-
-  def run(self):
-    if not self.protoc_command:
-      raise CommandError('could not find protoc')
-    if not self.grpc_python_plugin_command:
-      raise CommandError('could not find grpc_python_plugin '
-                         '(protoc plugin for GRPC Python)')
-
-    if not os.path.exists(PROTO_GEN_STEM):
-      os.makedirs(PROTO_GEN_STEM)
-
-    include_regex = re.compile(self.include)
-    exclude_regex = re.compile(self.exclude) if self.exclude else None
-    paths = []
-    for walk_root, directories, filenames in os.walk(PROTO_STEM):
-      for filename in filenames:
-        path = os.path.join(walk_root, filename)
-        if include_regex.match(path) and not (
-            exclude_regex and exclude_regex.match(path)):
-          paths.append(path)
-
-    # TODO(kpayson): It would be nice to do this in a batch command,
-    # but we currently have name conflicts in src/proto
-    for path in paths:
-      command = [
-          self.protoc_command,
-          '--plugin=protoc-gen-python-grpc={}'.format(
-              self.grpc_python_plugin_command),
-          '-I {}'.format(GRPC_STEM),
-          '-I .',
-          '-I {}/third_party/protobuf/src'.format(GRPC_STEM),
-          '--python_out={}'.format(PROTO_GEN_STEM),
-          '--python-grpc_out={}'.format(PROTO_GEN_STEM),
-      ] + [path]
-      try:
-        subprocess.check_output(' '.join(command), cwd=PYTHON_STEM, shell=True,
-                                stderr=subprocess.STDOUT)
-      except subprocess.CalledProcessError as e:
-        sys.stderr.write(
-            'warning: Command:\n{}\nMessage:\n{}\nOutput:\n{}'.format(
-                command, str(e), e.output))
-
-    # Generated proto directories dont include __init__.py, but
-    # these are needed for python package resolution
-    for walk_root, _, _ in os.walk(PROTO_GEN_STEM):
-      if walk_root != PROTO_GEN_STEM:
-        path = os.path.join(walk_root, '__init__.py')
-        open(path, 'a').close()
-
-
 class BuildProjectMetadata(setuptools.Command):
   """Command to generate project metadata in a module."""
 
@@ -225,10 +156,6 @@ class BuildPy(build_py.build_py):
   """Custom project build command."""
 
   def run(self):
-    try:
-      self.run_command('build_proto_modules')
-    except CommandError as error:
-      sys.stderr.write('warning: %s\n' % error.message)
     self.run_command('build_project_metadata')
     build_py.build_py.run(self)
 
@@ -281,76 +208,3 @@ class Gather(setuptools.Command):
       self.distribution.fetch_build_eggs(self.distribution.install_requires)
     if self.test and self.distribution.tests_require:
       self.distribution.fetch_build_eggs(self.distribution.tests_require)
-
-
-class TestLite(setuptools.Command):
-  """Command to run tests without fetching or building anything."""
-
-  description = 'run tests without fetching or building anything.'
-  user_options = []
-
-  def initialize_options(self):
-    pass
-
-  def finalize_options(self):
-    # distutils requires this override.
-    pass
-
-  def run(self):
-    self._add_eggs_to_path()
-
-    import tests
-    loader = tests.Loader()
-    loader.loadTestsFromNames(['tests'])
-    runner = tests.Runner()
-    result = runner.run(loader.suite)
-    if not result.wasSuccessful():
-      sys.exit('Test failure')
-
-  def _add_eggs_to_path(self):
-    """Fetch install and test requirements"""
-    self.distribution.fetch_build_eggs(self.distribution.install_requires)
-    self.distribution.fetch_build_eggs(self.distribution.tests_require)
-
-
-class RunInterop(test.test):
-
-  description = 'run interop test client/server'
-  user_options = [
-    ('args=', 'a', 'pass-thru arguments for the client/server'),
-    ('client', 'c', 'flag indicating to run the client'),
-    ('server', 's', 'flag indicating to run the server')
-  ]
-
-  def initialize_options(self):
-    self.args = ''
-    self.client = False
-    self.server = False
-
-  def finalize_options(self):
-    if self.client and self.server:
-      raise DistutilsOptionError('you may only specify one of client or server')
-
-  def run(self):
-    if self.distribution.install_requires:
-      self.distribution.fetch_build_eggs(self.distribution.install_requires)
-    if self.distribution.tests_require:
-      self.distribution.fetch_build_eggs(self.distribution.tests_require)
-    if self.client:
-      self.run_client()
-    elif self.server:
-      self.run_server()
-
-  def run_server(self):
-    # We import here to ensure that our setuptools parent has had a chance to
-    # edit the Python system path.
-    from tests.interop import server
-    sys.argv[1:] = self.args.split()
-    server.serve()
-
-  def run_client(self):
-    # We import here to ensure that our setuptools parent has had a chance to
-    # edit the Python system path.
-    from tests.interop import client
-    sys.argv[1:] = self.args.split()
-    client.test_interoperability()

src/python/grpcio_health_checking/health_commands.py

@@ -39,51 +39,17 @@ import sys
 
 import setuptools
 from setuptools.command import build_py
-from setuptools.command import sdist
 
 ROOT_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__)))
 HEALTH_PROTO = os.path.join(ROOT_DIR, '../../proto/grpc/health/v1/health.proto')
 
 
-class BuildProtoModules(setuptools.Command):
+class CopyProtoModules(setuptools.Command):
-  """Command to generate project *_pb2.py modules from proto files."""
+  """Command to copy proto modules from grpc/src/proto."""
 
   description = ''
   user_options = []
 
-  def initialize_options(self):
-    pass
-
-  def finalize_options(self):
-    self.protoc_command = distutils.spawn.find_executable('protoc')
-    self.grpc_python_plugin_command = distutils.spawn.find_executable(
-        'grpc_python_plugin')
-
-  def run(self):
-    paths = []
-    root_directory = os.getcwd()
-    for walk_root, directories, filenames in os.walk(root_directory):
-      for filename in filenames:
-        if filename.endswith('.proto'):
-          paths.append(os.path.join(walk_root, filename))
-    command = [
-        self.protoc_command,
-        '--plugin=protoc-gen-python-grpc={}'.format(
-            self.grpc_python_plugin_command),
-        '-I {}'.format(root_directory),
-        '--python_out={}'.format(root_directory),
-        '--python-grpc_out={}'.format(root_directory),
-    ] + paths
-    try:
-      subprocess.check_output(' '.join(command), cwd=root_directory, shell=True,
-                              stderr=subprocess.STDOUT)
-    except subprocess.CalledProcessError as e:
-      raise Exception('{}\nOutput:\n{}'.format(e.message, e.output))
-
-
-class CopyProtoModules(setuptools.Command):
-  """Command to copy proto modules from grpc/src/proto."""
-
   def initialize_options(self):
     pass
 
@@ -101,14 +67,5 @@ class BuildPy(build_py.build_py):
   """Custom project build command."""
 
   def run(self):
-    self.run_command('copy_proto_modules')
     self.run_command('build_proto_modules')
     build_py.build_py.run(self)
-
-
-class SDist(sdist.sdist):
-  """Custom project build command."""
-
-  def run(self):
-    self.run_command('copy_proto_modules')
-    sdist.sdist.run(self)

src/python/grpcio_health_checking/setup.py

@@ -36,36 +36,44 @@ import sys
 from distutils import core as _core
 import setuptools
 
+import grpc.tools.command
+
 # Ensure we're in the proper directory whether or not we're being used by pip.
 os.chdir(os.path.dirname(os.path.abspath(__file__)))
 
 # Break import-style to ensure we can actually find our commands module.
 import health_commands
 
-_PACKAGES = (
+PACKAGES = (
     setuptools.find_packages('.')
 )
 
-_PACKAGE_DIRECTORIES = {
+PACKAGE_DIRECTORIES = {
     '': '.',
 }
 
-_INSTALL_REQUIRES = (
+SETUP_REQUIRES = (
+    'grpcio-tools>=0.14.0',
+)
+
+INSTALL_REQUIRES = (
     'grpcio>=0.13.1',
 )
 
-_COMMAND_CLASS = {
+COMMAND_CLASS = {
-    'copy_proto_modules': health_commands.CopyProtoModules,
+    # Run preprocess from the repository *before* doing any packaging!
-    'build_proto_modules': health_commands.BuildProtoModules,
+    'preprocess': health_commands.CopyProtoModules,
+
+    'build_proto_modules': grpc.tools.command.BuildProtoModules,
     'build_py': health_commands.BuildPy,
-    'sdist': health_commands.SDist,
 }
 
 setuptools.setup(
-    name='grpcio_health_checking',
+    name='grpcio-health-checking',
-    version='0.14.0b0',
+    version='0.14.0',
-    packages=list(_PACKAGES),
+    packages=list(PACKAGES),
-    package_dir=_PACKAGE_DIRECTORIES,
+    package_dir=PACKAGE_DIRECTORIES,
-    install_requires=_INSTALL_REQUIRES,
+    install_requires=INSTALL_REQUIRES,
-    cmdclass=_COMMAND_CLASS
+    setup_requires=SETUP_REQUIRES,
+    cmdclass=COMMAND_CLASS
 )

src/python/grpcio_tests/.gitignore

@@ -0,0 +1,4 @@
+proto/
+src/
+*_pb2.py
+*.egg-info/

src/python/grpcio_tests/commands.py

@@ -0,0 +1,217 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Provides distutils command classes for the gRPC Python setup process."""
+
+import distutils
+import glob
+import os
+import os.path
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import traceback
+
+import setuptools
+from setuptools.command import build_ext
+from setuptools.command import build_py
+from setuptools.command import easy_install
+from setuptools.command import install
+from setuptools.command import test
+
+PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
+GRPC_STEM = os.path.abspath(PYTHON_STEM + '../../../../')
+GRPC_PROTO_STEM = os.path.join(GRPC_STEM, 'src', 'proto')
+PROTO_STEM = os.path.join(PYTHON_STEM, 'src', 'proto')
+PYTHON_PROTO_TOP_LEVEL = os.path.join(PYTHON_STEM, 'src')
+
+
+class CommandError(object):
+  pass
+
+
+class GatherProto(setuptools.Command):
+
+  description = 'gather proto dependencies'
+  user_options = []
+
+  def initialize_options(self):
+    pass
+
+  def finalize_options(self):
+    pass
+
+  def run(self):
+    # TODO(atash) ensure that we're running from the repository directory when
+    # this command is used
+    try:
+      shutil.rmtree(PROTO_STEM)
+    except Exception as error:
+      # We don't care if this command fails
+      pass
+    shutil.copytree(GRPC_PROTO_STEM, PROTO_STEM)
+    for root, _, _ in os.walk(PYTHON_PROTO_TOP_LEVEL):
+      path = os.path.join(root, '__init__.py')
+      open(path, 'a').close()
+
+
+class BuildProtoModules(setuptools.Command):
+  """Command to generate project *_pb2.py modules from proto files."""
+
+  description = 'build protobuf modules'
+  user_options = [
+    ('include=', None, 'path patterns to include in protobuf generation'),
+    ('exclude=', None, 'path patterns to exclude from protobuf generation')
+  ]
+
+  def initialize_options(self):
+    self.exclude = None
+    self.include = r'.*\.proto$'
+
+  def finalize_options(self):
+    pass
+
+  def run(self):
+    import grpc.tools.protoc as protoc
+
+    include_regex = re.compile(self.include)
+    exclude_regex = re.compile(self.exclude) if self.exclude else None
+    paths = []
+    for walk_root, directories, filenames in os.walk(PROTO_STEM):
+      for filename in filenames:
+        path = os.path.join(walk_root, filename)
+        if include_regex.match(path) and not (
+            exclude_regex and exclude_regex.match(path)):
+          paths.append(path)
+
+    # TODO(kpayson): It would be nice to do this in a batch command,
+    # but we currently have name conflicts in src/proto
+    for path in paths:
+      command = [
+          'grpc.tools.protoc',
+          '-I {}'.format(PROTO_STEM),
+          '--python_out={}'.format(PROTO_STEM),
+          '--grpc_python_out={}'.format(PROTO_STEM),
+      ] + [path]
+      if protoc.main(command) != 0:
+        sys.stderr.write(
+            'warning: Command:\n{}\nFailed'.format(
+                command))
+
+    # Generated proto directories dont include __init__.py, but
+    # these are needed for python package resolution
+    for walk_root, _, _ in os.walk(PROTO_STEM):
+      path = os.path.join(walk_root, '__init__.py')
+      open(path, 'a').close()
+
+
+class BuildPy(build_py.build_py):
+  """Custom project build command."""
+
+  def run(self):
+    try:
+      self.run_command('build_proto_modules')
+    except CommandError as error:
+      sys.stderr.write('warning: %s\n' % error.message)
+    build_py.build_py.run(self)
+
+
+class TestLite(setuptools.Command):
+  """Command to run tests without fetching or building anything."""
+
+  description = 'run tests without fetching or building anything.'
+  user_options = []
+
+  def initialize_options(self):
+    pass
+
+  def finalize_options(self):
+    # distutils requires this override.
+    pass
+
+  def run(self):
+    self._add_eggs_to_path()
+
+    import tests
+    loader = tests.Loader()
+    loader.loadTestsFromNames(['tests'])
+    runner = tests.Runner()
+    result = runner.run(loader.suite)
+    if not result.wasSuccessful():
+      sys.exit('Test failure')
+
+  def _add_eggs_to_path(self):
+    """Fetch install and test requirements"""
+    self.distribution.fetch_build_eggs(self.distribution.install_requires)
+    self.distribution.fetch_build_eggs(self.distribution.tests_require)
+
+
+class RunInterop(test.test):
+
+  description = 'run interop test client/server'
+  user_options = [
+    ('args=', 'a', 'pass-thru arguments for the client/server'),
+    ('client', 'c', 'flag indicating to run the client'),
+    ('server', 's', 'flag indicating to run the server')
+  ]
+
+  def initialize_options(self):
+    self.args = ''
+    self.client = False
+    self.server = False
+
+  def finalize_options(self):
+    if self.client and self.server:
+      raise DistutilsOptionError('you may only specify one of client or server')
+
+  def run(self):
+    if self.distribution.install_requires:
+      self.distribution.fetch_build_eggs(self.distribution.install_requires)
+    if self.distribution.tests_require:
+      self.distribution.fetch_build_eggs(self.distribution.tests_require)
+    if self.client:
+      self.run_client()
+    elif self.server:
+      self.run_server()
+
+  def run_server(self):
+    # We import here to ensure that our setuptools parent has had a chance to
+    # edit the Python system path.
+    from tests.interop import server
+    sys.argv[1:] = self.args.split()
+    server.serve()
+
+  def run_client(self):
+    # We import here to ensure that our setuptools parent has had a chance to
+    # edit the Python system path.
+    from tests.interop import client
+    sys.argv[1:] = self.args.split()
+    client.test_interoperability()

src/python/grpcio_tests/grpc_version.py

@@ -0,0 +1,32 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!!
+
+VERSION='0.15.0.dev0'

src/python/grpcio_tests/setup.py

@@ -0,0 +1,124 @@
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""A setup module for the gRPC Python package."""
+
+import os
+import os.path
+import shutil
+import sys
+
+from distutils import core as _core
+from distutils import extension as _extension
+import setuptools
+from setuptools.command import egg_info
+
+import grpc.tools.command
+
+PY3 = sys.version_info.major == 3
+
+# Ensure we're in the proper directory whether or not we're being used by pip.
+os.chdir(os.path.dirname(os.path.abspath(__file__)))
+
+# Break import-style to ensure we can actually find our in-repo dependencies.
+import commands
+import grpc_version
+
+LICENSE = '3-clause BSD'
+
+PACKAGE_DIRECTORIES = {
+    '': '.',
+}
+
+INSTALL_REQUIRES = (
+    'coverage>=4.0',
+    'enum34>=1.0.4',
+    'futures>=2.2.0',
+    'grpcio>=0.14.0',
+    'grpcio-health-checking>=0.14.0',
+    'oauth2client>=1.4.7',
+    'protobuf>=3.0.0a3',
+    'six>=1.10',
+)
+
+SETUP_REQUIRES = (
+    'grpcio-tools>=0.14.0',
+)
+
+COMMAND_CLASS = {
+    # Run `preprocess` *before* doing any packaging!
+    'preprocess': commands.GatherProto,
+
+    'build_proto_modules': grpc.tools.command.BuildProtoModules,
+    'build_py': commands.BuildPy,
+    'run_interop': commands.RunInterop,
+    'test_lite': commands.TestLite
+}
+
+PACKAGE_DATA = {
+    'tests.interop': [
+        'credentials/ca.pem',
+        'credentials/server1.key',
+        'credentials/server1.pem',
+    ],
+    'tests.protoc_plugin': [
+        'protoc_plugin_test.proto',
+    ],
+    'tests.unit': [
+        'credentials/ca.pem',
+        'credentials/server1.key',
+        'credentials/server1.pem',
+    ],
+    'tests': [
+        'tests.json'
+    ],
+}
+
+TEST_SUITE = 'tests'
+TEST_LOADER = 'tests:Loader'
+TEST_RUNNER = 'tests:Runner'
+TESTS_REQUIRE = INSTALL_REQUIRES
+
+PACKAGES = setuptools.find_packages('.')
+
+setuptools.setup(
+  name='grpcio-tests',
+  version=grpc_version.VERSION,
+  license=LICENSE,
+  packages=list(PACKAGES),
+  package_dir=PACKAGE_DIRECTORIES,
+  package_data=PACKAGE_DATA,
+  install_requires=INSTALL_REQUIRES,
+  setup_requires=SETUP_REQUIRES,
+  cmdclass=COMMAND_CLASS,
+  tests_require=TESTS_REQUIRE,
+  test_suite=TEST_SUITE,
+  test_loader=TEST_LOADER,
+  test_runner=TEST_RUNNER,
+)

src/python/grpcio_tests/tests/protoc_plugin/_python_plugin_test.py

@@ -45,6 +45,11 @@ from six import moves
 import grpc
 from tests.unit.framework.common import test_constants
 
+import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2
+import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2
+import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2
+import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2
+
 # Identifiers of entities we expect to find in the generated module.
 STUB_IDENTIFIER = 'TestServiceStub'
 SERVICER_IDENTIFIER = 'TestServiceServicer'
@@ -53,12 +58,10 @@ ADD_SERVICER_TO_SERVER_IDENTIFIER = 'add_TestServiceServicer_to_server'
 
 class _ServicerMethods(object):
 
-  def __init__(self, response_pb2, payload_pb2):
+  def __init__(self):
     self._condition = threading.Condition()
     self._paused = False
     self._fail = False
-    self._response_pb2 = response_pb2
-    self._payload_pb2 = payload_pb2
 
   @contextlib.contextmanager
   def pause(self):  # pylint: disable=invalid-name
@@ -85,22 +88,22 @@ class _ServicerMethods(object):
         self._condition.wait()
 
   def UnaryCall(self, request, unused_rpc_context):
-    response = self._response_pb2.SimpleResponse()
+    response = response_pb2.SimpleResponse()
-    response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+    response.payload.payload_type = payload_pb2.COMPRESSABLE
     response.payload.payload_compressable = 'a' * request.response_size
     self._control()
     return response
 
   def StreamingOutputCall(self, request, unused_rpc_context):
     for parameter in request.response_parameters:
-      response = self._response_pb2.StreamingOutputCallResponse()
+      response = response_pb2.StreamingOutputCallResponse()
-      response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+      response.payload.payload_type = payload_pb2.COMPRESSABLE
       response.payload.payload_compressable = 'a' * parameter.size
       self._control()
       yield response
 
   def StreamingInputCall(self, request_iter, unused_rpc_context):
-    response = self._response_pb2.StreamingInputCallResponse()
+    response = response_pb2.StreamingInputCallResponse()
     aggregated_payload_size = 0
     for request in request_iter:
       aggregated_payload_size += len(request.payload.payload_compressable)
@@ -111,8 +114,8 @@ class _ServicerMethods(object):
   def FullDuplexCall(self, request_iter, unused_rpc_context):
     for request in request_iter:
       for parameter in request.response_parameters:
-        response = self._response_pb2.StreamingOutputCallResponse()
+        response = response_pb2.StreamingOutputCallResponse()
-        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+        response.payload.payload_type = payload_pb2.COMPRESSABLE
         response.payload.payload_compressable = 'a' * parameter.size
         self._control()
         yield response
@@ -121,8 +124,8 @@ class _ServicerMethods(object):
     responses = []
     for request in request_iter:
       for parameter in request.response_parameters:
-        response = self._response_pb2.StreamingOutputCallResponse()
+        response = response_pb2.StreamingOutputCallResponse()
-        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+        response.payload.payload_type = payload_pb2.COMPRESSABLE
         response.payload.payload_compressable = 'a' * parameter.size
         self._control()
         responses.append(response)
@@ -142,18 +145,13 @@ class _Service(
   """
       
 
-def _CreateService(service_pb2, response_pb2, payload_pb2):
+def _CreateService():
   """Provides a servicer backend and a stub.
 
-  Args:
-    service_pb2: The service_pb2 module generated by this test.
-    response_pb2: The response_pb2 module generated by this test.
-    payload_pb2: The payload_pb2 module generated by this test.
-
   Returns:
     A _Service with which to test RPCs.
   """
-  servicer_methods = _ServicerMethods(response_pb2, payload_pb2)
+  servicer_methods = _ServicerMethods()
 
   class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)):
 
@@ -182,12 +180,9 @@ def _CreateService(service_pb2, response_pb2, payload_pb2):
   return _Service(servicer_methods, server, stub)
 
 
-def _CreateIncompleteService(service_pb2):
+def _CreateIncompleteService():
   """Provides a servicer backend that fails to implement methods and its stub.
 
-  Args:
-    service_pb2: The service_pb2 module generated by this test.
-
   Returns:
     A _Service with which to test RPCs. The returned _Service's
       servicer_methods implements none of the methods required of it.
@@ -206,7 +201,7 @@ def _CreateIncompleteService(service_pb2):
   return _Service(None, server, stub)
 
 
-def _streaming_input_request_iterator(request_pb2, payload_pb2):
+def _streaming_input_request_iterator():
   for _ in range(3):
     request = request_pb2.StreamingInputCallRequest()
     request.payload.payload_type = payload_pb2.COMPRESSABLE
@@ -214,7 +209,7 @@ def _streaming_input_request_iterator(request_pb2, payload_pb2):
     yield request
 
 
-def _streaming_output_request(request_pb2):
+def _streaming_output_request():
   request = request_pb2.StreamingOutputCallRequest()
   sizes = [1, 2, 3]
   request.response_parameters.add(size=sizes[0], interval_us=0)
@@ -223,7 +218,7 @@ def _streaming_output_request(request_pb2):
   return request
 
 
-def _full_duplex_request_iterator(request_pb2):
+def _full_duplex_request_iterator():
   request = request_pb2.StreamingOutputCallRequest()
   request.response_parameters.add(size=1, interval_us=0)
   yield request
@@ -241,102 +236,40 @@ class PythonPluginTest(unittest.TestCase):
   methods and does not exist for response-streaming methods.
   """
 
-  def setUp(self):
-    # Assume that the appropriate protoc and grpc_python_plugins are on the
-    # path.
-    protoc_command = 'protoc'
-    protoc_plugin_filename = distutils.spawn.find_executable(
-        'grpc_python_plugin')
-    if not os.path.isfile(protoc_command):
-      # Assume that if we haven't built protoc that it's on the system.
-      protoc_command = 'protoc'
-
-    # Ensure that the output directory exists.
-    self.outdir = tempfile.mkdtemp()
-
-    # Find all proto files
-    paths = []
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    proto_dir = os.path.join(root_dir, 'protos')
-    for walk_root, _, filenames in os.walk(proto_dir):
-      for filename in filenames:
-        if filename.endswith('.proto'):
-          path = os.path.join(walk_root, filename)
-          paths.append(path)
-
-    # Invoke protoc with the plugin.
-    cmd = [
-        protoc_command,
-        '--plugin=protoc-gen-python-grpc=%s' % protoc_plugin_filename,
-        '-I %s' % root_dir,
-        '--python_out=%s' % self.outdir,
-        '--python-grpc_out=%s' % self.outdir
-    ] + paths
-    subprocess.check_call(' '.join(cmd), shell=True, env=os.environ,
-                          cwd=os.path.dirname(os.path.realpath(__file__)))
-
-    # Generated proto directories dont include __init__.py, but
-    # these are needed for python package resolution
-    for walk_root, _, _ in os.walk(os.path.join(self.outdir, 'protos')):
-      path = os.path.join(walk_root, '__init__.py')
-      open(path, 'a').close()
-
-    sys.path.insert(0, self.outdir)
-
-    import protos.payload.test_payload_pb2 as payload_pb2
-    import protos.requests.r.test_requests_pb2 as request_pb2
-    import protos.responses.test_responses_pb2 as response_pb2
-    import protos.service.test_service_pb2 as service_pb2
-    self._payload_pb2 = payload_pb2
-    self._request_pb2 = request_pb2
-    self._response_pb2 = response_pb2
-    self._service_pb2 = service_pb2
-
-  def tearDown(self):
-    try:
-      shutil.rmtree(self.outdir)
-    except OSError as exc:
-      if exc.errno != errno.ENOENT:
-        raise
-    sys.path.remove(self.outdir)
-
   def testImportAttributes(self):
     # check that we can access the generated module and its members.
     self.assertIsNotNone(
-        getattr(self._service_pb2, STUB_IDENTIFIER, None))
+        getattr(service_pb2, STUB_IDENTIFIER, None))
     self.assertIsNotNone(
-        getattr(self._service_pb2, SERVICER_IDENTIFIER, None))
+        getattr(service_pb2, SERVICER_IDENTIFIER, None))
     self.assertIsNotNone(
-        getattr(self._service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER, None))
+        getattr(service_pb2, ADD_SERVICER_TO_SERVER_IDENTIFIER, None))
 
   def testUpDown(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     self.assertIsNotNone(service.servicer_methods)
     self.assertIsNotNone(service.server)
     self.assertIsNotNone(service.stub)
 
   def testIncompleteServicer(self):
-    service = _CreateIncompleteService(self._service_pb2)
+    service = _CreateIncompleteService()
-    request = self._request_pb2.SimpleRequest(response_size=13)
+    request = request_pb2.SimpleRequest(response_size=13)
     with self.assertRaises(grpc.RpcError) as exception_context:
       service.stub.UnaryCall(request)
     self.assertIs(
         exception_context.exception.code(), grpc.StatusCode.UNIMPLEMENTED)
 
   def testUnaryCall(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = request_pb2.SimpleRequest(response_size=13)
-    request = self._request_pb2.SimpleRequest(response_size=13)
     response = service.stub.UnaryCall(request)
     expected_response = service.servicer_methods.UnaryCall(
         request, 'not a real context!')
     self.assertEqual(expected_response, response)
 
   def testUnaryCallFuture(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = request_pb2.SimpleRequest(response_size=13)
-    request = self._request_pb2.SimpleRequest(response_size=13)
     # Check that the call does not block waiting for the server to respond.
     with service.servicer_methods.pause():
       response_future = service.stub.UnaryCall.future(request)
@@ -346,9 +279,8 @@ class PythonPluginTest(unittest.TestCase):
     self.assertEqual(expected_response, response)
 
   def testUnaryCallFutureExpired(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = request_pb2.SimpleRequest(response_size=13)
-    request = self._request_pb2.SimpleRequest(response_size=13)
     with service.servicer_methods.pause():
       response_future = service.stub.UnaryCall.future(
           request, timeout=test_constants.SHORT_TIMEOUT)
@@ -359,9 +291,8 @@ class PythonPluginTest(unittest.TestCase):
     self.assertIs(response_future.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
 
   def testUnaryCallFutureCancelled(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = request_pb2.SimpleRequest(response_size=13)
-    request = self._request_pb2.SimpleRequest(response_size=13)
     with service.servicer_methods.pause():
       response_future = service.stub.UnaryCall.future(request)
       response_future.cancel()
@@ -369,18 +300,16 @@ class PythonPluginTest(unittest.TestCase):
     self.assertIs(response_future.code(), grpc.StatusCode.CANCELLED)
 
   def testUnaryCallFutureFailed(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = request_pb2.SimpleRequest(response_size=13)
-    request = self._request_pb2.SimpleRequest(response_size=13)
     with service.servicer_methods.fail():
       response_future = service.stub.UnaryCall.future(request)
       self.assertIsNotNone(response_future.exception())
     self.assertIs(response_future.code(), grpc.StatusCode.UNKNOWN)
 
   def testStreamingOutputCall(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request()
-    request = _streaming_output_request(self._request_pb2)
     responses = service.stub.StreamingOutputCall(request)
     expected_responses = service.servicer_methods.StreamingOutputCall(
         request, 'not a real RpcContext!')
@@ -389,9 +318,8 @@ class PythonPluginTest(unittest.TestCase):
       self.assertEqual(expected_response, response)
 
   def testStreamingOutputCallExpired(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request()
-    request = _streaming_output_request(self._request_pb2)
     with service.servicer_methods.pause():
       responses = service.stub.StreamingOutputCall(
           request, timeout=test_constants.SHORT_TIMEOUT)
@@ -401,9 +329,8 @@ class PythonPluginTest(unittest.TestCase):
         exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
 
   def testStreamingOutputCallCancelled(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request()
-    request = _streaming_output_request(self._request_pb2)
     responses = service.stub.StreamingOutputCall(request)
     next(responses)
     responses.cancel()
@@ -412,9 +339,8 @@ class PythonPluginTest(unittest.TestCase):
     self.assertIs(responses.code(), grpc.StatusCode.CANCELLED)
 
   def testStreamingOutputCallFailed(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request = _streaming_output_request()
-    request = _streaming_output_request(self._request_pb2)
     with service.servicer_methods.fail():
       responses = service.stub.StreamingOutputCall(request)
       self.assertIsNotNone(responses)
@@ -423,36 +349,30 @@ class PythonPluginTest(unittest.TestCase):
     self.assertIs(exception_context.exception.code(), grpc.StatusCode.UNKNOWN)
 
   def testStreamingInputCall(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     response = service.stub.StreamingInputCall(
-        _streaming_input_request_iterator(
+        _streaming_input_request_iterator())
-            self._request_pb2, self._payload_pb2))
     expected_response = service.servicer_methods.StreamingInputCall(
-        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        _streaming_input_request_iterator(),
         'not a real RpcContext!')
     self.assertEqual(expected_response, response)
 
   def testStreamingInputCallFuture(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with service.servicer_methods.pause():
       response_future = service.stub.StreamingInputCall.future(
-          _streaming_input_request_iterator(
+          _streaming_input_request_iterator())
-              self._request_pb2, self._payload_pb2))
     response = response_future.result()
     expected_response = service.servicer_methods.StreamingInputCall(
-        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        _streaming_input_request_iterator(),
         'not a real RpcContext!')
     self.assertEqual(expected_response, response)
 
   def testStreamingInputCallFutureExpired(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with service.servicer_methods.pause():
       response_future = service.stub.StreamingInputCall.future(
-          _streaming_input_request_iterator(
+          _streaming_input_request_iterator(),
-              self._request_pb2, self._payload_pb2),
           timeout=test_constants.SHORT_TIMEOUT)
       with self.assertRaises(grpc.RpcError) as exception_context:
         response_future.result()
@@ -463,43 +383,37 @@ class PythonPluginTest(unittest.TestCase):
         exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
 
   def testStreamingInputCallFutureCancelled(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with service.servicer_methods.pause():
       response_future = service.stub.StreamingInputCall.future(
-          _streaming_input_request_iterator(
+          _streaming_input_request_iterator())
-              self._request_pb2, self._payload_pb2))
       response_future.cancel()
     self.assertTrue(response_future.cancelled())
     with self.assertRaises(grpc.FutureCancelledError):
       response_future.result()
 
   def testStreamingInputCallFutureFailed(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with service.servicer_methods.fail():
       response_future = service.stub.StreamingInputCall.future(
-          _streaming_input_request_iterator(
+          _streaming_input_request_iterator())
-              self._request_pb2, self._payload_pb2))
       self.assertIsNotNone(response_future.exception())
       self.assertIs(response_future.code(), grpc.StatusCode.UNKNOWN)
 
   def testFullDuplexCall(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     responses = service.stub.FullDuplexCall(
-        _full_duplex_request_iterator(self._request_pb2))
+        _full_duplex_request_iterator())
     expected_responses = service.servicer_methods.FullDuplexCall(
-        _full_duplex_request_iterator(self._request_pb2),
+        _full_duplex_request_iterator(),
         'not a real RpcContext!')
     for expected_response, response in moves.zip_longest(
         expected_responses, responses):
       self.assertEqual(expected_response, response)
 
   def testFullDuplexCallExpired(self):
-    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    request_iterator = _full_duplex_request_iterator()
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with service.servicer_methods.pause():
       responses = service.stub.FullDuplexCall(
           request_iterator, timeout=test_constants.SHORT_TIMEOUT)
@@ -509,9 +423,8 @@ class PythonPluginTest(unittest.TestCase):
         exception_context.exception.code(), grpc.StatusCode.DEADLINE_EXCEEDED)
 
   def testFullDuplexCallCancelled(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
+    request_iterator = _full_duplex_request_iterator()
-    request_iterator = _full_duplex_request_iterator(self._request_pb2)
     responses = service.stub.FullDuplexCall(request_iterator)
     next(responses)
     responses.cancel()
@@ -521,9 +434,8 @@ class PythonPluginTest(unittest.TestCase):
         exception_context.exception.code(), grpc.StatusCode.CANCELLED)
 
   def testFullDuplexCallFailed(self):
-    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    request_iterator = _full_duplex_request_iterator()
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with service.servicer_methods.fail():
       responses = service.stub.FullDuplexCall(request_iterator)
       with self.assertRaises(grpc.RpcError) as exception_context:
@@ -531,13 +443,12 @@ class PythonPluginTest(unittest.TestCase):
     self.assertIs(exception_context.exception.code(), grpc.StatusCode.UNKNOWN)
 
   def testHalfDuplexCall(self):
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     def half_duplex_request_iterator():
-      request = self._request_pb2.StreamingOutputCallRequest()
+      request = request_pb2.StreamingOutputCallRequest()
       request.response_parameters.add(size=1, interval_us=0)
       yield request
-      request = self._request_pb2.StreamingOutputCallRequest()
+      request = request_pb2.StreamingOutputCallRequest()
       request.response_parameters.add(size=2, interval_us=0)
       request.response_parameters.add(size=3, interval_us=0)
       yield request
@@ -561,14 +472,13 @@ class PythonPluginTest(unittest.TestCase):
         wait_cell[0] = False
         condition.notify_all()
     def half_duplex_request_iterator():
-      request = self._request_pb2.StreamingOutputCallRequest()
+      request = request_pb2.StreamingOutputCallRequest()
       request.response_parameters.add(size=1, interval_us=0)
       yield request
       with condition:
         while wait_cell[0]:
           condition.wait()
-    service = _CreateService(
+    service = _CreateService()
-        self._service_pb2, self._response_pb2, self._payload_pb2)
     with wait():
       responses = service.stub.HalfDuplexCall(
           half_duplex_request_iterator(), timeout=test_constants.SHORT_TIMEOUT)

src/python/grpcio_tests/tests/protoc_plugin/beta_python_plugin_test.py

@@ -50,6 +50,11 @@ from grpc.framework.foundation import future
 from grpc.framework.interfaces.face import face
 from tests.unit.framework.common import test_constants
 
+import tests.protoc_plugin.protos.payload.test_payload_pb2 as payload_pb2
+import tests.protoc_plugin.protos.requests.r.test_requests_pb2 as request_pb2
+import tests.protoc_plugin.protos.responses.test_responses_pb2 as response_pb2
+import tests.protoc_plugin.protos.service.test_service_pb2 as service_pb2
+
 # Identifiers of entities we expect to find in the generated module.
 SERVICER_IDENTIFIER = 'BetaTestServiceServicer'
 STUB_IDENTIFIER = 'BetaTestServiceStub'
@@ -59,12 +64,10 @@ STUB_FACTORY_IDENTIFIER = 'beta_create_TestService_stub'
 
 class _ServicerMethods(object):
 
-  def __init__(self, response_pb2, payload_pb2):
+  def __init__(self):
     self._condition = threading.Condition()
     self._paused = False
     self._fail = False
-    self._response_pb2 = response_pb2
-    self._payload_pb2 = payload_pb2
 
   @contextlib.contextmanager
   def pause(self):  # pylint: disable=invalid-name
@@ -91,22 +94,22 @@ class _ServicerMethods(object):
         self._condition.wait()
 
   def UnaryCall(self, request, unused_rpc_context):
-    response = self._response_pb2.SimpleResponse()
+    response = response_pb2.SimpleResponse()
-    response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+    response.payload.payload_type = payload_pb2.COMPRESSABLE
     response.payload.payload_compressable = 'a' * request.response_size
     self._control()
     return response
 
   def StreamingOutputCall(self, request, unused_rpc_context):
     for parameter in request.response_parameters:
-      response = self._response_pb2.StreamingOutputCallResponse()
+      response = response_pb2.StreamingOutputCallResponse()
-      response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+      response.payload.payload_type = payload_pb2.COMPRESSABLE
       response.payload.payload_compressable = 'a' * parameter.size
       self._control()
       yield response
 
   def StreamingInputCall(self, request_iter, unused_rpc_context):
-    response = self._response_pb2.StreamingInputCallResponse()
+    response = response_pb2.StreamingInputCallResponse()
     aggregated_payload_size = 0
     for request in request_iter:
       aggregated_payload_size += len(request.payload.payload_compressable)
@@ -117,8 +120,8 @@ class _ServicerMethods(object):
   def FullDuplexCall(self, request_iter, unused_rpc_context):
     for request in request_iter:
       for parameter in request.response_parameters:
-        response = self._response_pb2.StreamingOutputCallResponse()
+        response = response_pb2.StreamingOutputCallResponse()
-        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+        response.payload.payload_type = payload_pb2.COMPRESSABLE
         response.payload.payload_compressable = 'a' * parameter.size
         self._control()
         yield response
@@ -127,8 +130,8 @@ class _ServicerMethods(object):
     responses = []
     for request in request_iter:
       for parameter in request.response_parameters:
-        response = self._response_pb2.StreamingOutputCallResponse()
+        response = response_pb2.StreamingOutputCallResponse()
-        response.payload.payload_type = self._payload_pb2.COMPRESSABLE
+        response.payload.payload_type = payload_pb2.COMPRESSABLE
         response.payload.payload_compressable = 'a' * parameter.size
         self._control()
         responses.append(response)
@@ -137,23 +140,18 @@ class _ServicerMethods(object):
 
 
 @contextlib.contextmanager
-def _CreateService(service_pb2, response_pb2, payload_pb2):
+def _CreateService():
   """Provides a servicer backend and a stub.
 
   The servicer is just the implementation of the actual servicer passed to the
   face player of the python RPC implementation; the two are detached.
 
-  Args:
-    service_pb2: The service_pb2 module generated by this test.
-    response_pb2: The response_pb2 module generated by this test
-    payload_pb2: The payload_pb2 module generated by this test
-
   Yields:
     A (servicer_methods, stub) pair where servicer_methods is the back-end of
       the service bound to the stub and and stub is the stub on which to invoke
       RPCs.
   """
-  servicer_methods = _ServicerMethods(response_pb2, payload_pb2)
+  servicer_methods = _ServicerMethods()
 
   class Servicer(getattr(service_pb2, SERVICER_IDENTIFIER)):
 
@@ -183,7 +181,7 @@ def _CreateService(service_pb2, response_pb2, payload_pb2):
 
 
 @contextlib.contextmanager
-def _CreateIncompleteService(service_pb2):
+def _CreateIncompleteService():
   """Provides a servicer backend that fails to implement methods and its stub.
 
   The servicer is just the implementation of the actual servicer passed to the
@@ -209,7 +207,7 @@ def _CreateIncompleteService(service_pb2):
   server.stop(0)
 
 
-def _streaming_input_request_iterator(request_pb2, payload_pb2):
+def _streaming_input_request_iterator():
   for _ in range(3):
     request = request_pb2.StreamingInputCallRequest()
     request.payload.payload_type = payload_pb2.COMPRESSABLE
@@ -217,7 +215,7 @@ def _streaming_input_request_iterator(request_pb2, payload_pb2):
     yield request
 
 
-def _streaming_output_request(request_pb2):
+def _streaming_output_request():
   request = request_pb2.StreamingOutputCallRequest()
   sizes = [1, 2, 3]
   request.response_parameters.add(size=sizes[0], interval_us=0)
@@ -226,7 +224,7 @@ def _streaming_output_request(request_pb2):
   return request
 
 
-def _full_duplex_request_iterator(request_pb2):
+def _full_duplex_request_iterator():
   request = request_pb2.StreamingOutputCallRequest()
   request.response_parameters.add(size=1, interval_us=0)
   yield request
@@ -244,101 +242,39 @@ class PythonPluginTest(unittest.TestCase):
   methods and does not exist for response-streaming methods.
   """
 
-  def setUp(self):
-    # Assume that the appropriate protoc and grpc_python_plugins are on the
-    # path.
-    protoc_command = 'protoc'
-    protoc_plugin_filename = distutils.spawn.find_executable(
-        'grpc_python_plugin')
-    if not os.path.isfile(protoc_command):
-      # Assume that if we haven't built protoc that it's on the system.
-      protoc_command = 'protoc'
-
-    # Ensure that the output directory exists.
-    self.outdir = tempfile.mkdtemp()
-
-    # Find all proto files
-    paths = []
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    proto_dir = os.path.join(root_dir, 'protos')
-    for walk_root, _, filenames in os.walk(proto_dir):
-      for filename in filenames:
-        if filename.endswith('.proto'):
-          path = os.path.join(walk_root, filename)
-          paths.append(path)
-
-    # Invoke protoc with the plugin.
-    cmd = [
-        protoc_command,
-        '--plugin=protoc-gen-python-grpc=%s' % protoc_plugin_filename,
-        '-I %s' % root_dir,
-        '--python_out=%s' % self.outdir,
-        '--python-grpc_out=%s' % self.outdir
-    ] + paths
-    subprocess.check_call(' '.join(cmd), shell=True, env=os.environ,
-                          cwd=os.path.dirname(os.path.realpath(__file__)))
-
-    # Generated proto directories dont include __init__.py, but
-    # these are needed for python package resolution
-    for walk_root, _, _ in os.walk(os.path.join(self.outdir, 'protos')):
-      path = os.path.join(walk_root, '__init__.py')
-      open(path, 'a').close()
-
-    sys.path.insert(0, self.outdir)
-
-    import protos.payload.test_payload_pb2 as payload_pb2  # pylint: disable=g-import-not-at-top
-    import protos.requests.r.test_requests_pb2 as request_pb2  # pylint: disable=g-import-not-at-top
-    import protos.responses.test_responses_pb2 as response_pb2  # pylint: disable=g-import-not-at-top
-    import protos.service.test_service_pb2 as service_pb2  # pylint: disable=g-import-not-at-top
-    self._payload_pb2 = payload_pb2
-    self._request_pb2 = request_pb2
-    self._response_pb2 = response_pb2
-    self._service_pb2 = service_pb2
-
-  def tearDown(self):
-    try:
-      shutil.rmtree(self.outdir)
-    except OSError as exc:
-      if exc.errno != errno.ENOENT:
-        raise
-    sys.path.remove(self.outdir)
-
   def testImportAttributes(self):
     # check that we can access the generated module and its members.
     self.assertIsNotNone(
-        getattr(self._service_pb2, SERVICER_IDENTIFIER, None))
+        getattr(service_pb2, SERVICER_IDENTIFIER, None))
     self.assertIsNotNone(
-        getattr(self._service_pb2, STUB_IDENTIFIER, None))
+        getattr(service_pb2, STUB_IDENTIFIER, None))
     self.assertIsNotNone(
-        getattr(self._service_pb2, SERVER_FACTORY_IDENTIFIER, None))
+        getattr(service_pb2, SERVER_FACTORY_IDENTIFIER, None))
     self.assertIsNotNone(
-        getattr(self._service_pb2, STUB_FACTORY_IDENTIFIER, None))
+        getattr(service_pb2, STUB_FACTORY_IDENTIFIER, None))
 
   def testUpDown(self):
-    with _CreateService(
+    with _CreateService():
-        self._service_pb2, self._response_pb2, self._payload_pb2):
+      request_pb2.SimpleRequest(response_size=13)
-      self._request_pb2.SimpleRequest(response_size=13)
 
   def testIncompleteServicer(self):
-    with _CreateIncompleteService(self._service_pb2) as (_, stub):
+    with _CreateIncompleteService() as (_, stub):
-      request = self._request_pb2.SimpleRequest(response_size=13)
+      request = request_pb2.SimpleRequest(response_size=13)
       try:
         stub.UnaryCall(request, test_constants.LONG_TIMEOUT)
       except face.AbortionError as error:
         self.assertEqual(interfaces.StatusCode.UNIMPLEMENTED, error.code)
 
   def testUnaryCall(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = request_pb2.SimpleRequest(response_size=13)
-      request = self._request_pb2.SimpleRequest(response_size=13)
       response = stub.UnaryCall(request, test_constants.LONG_TIMEOUT)
     expected_response = methods.UnaryCall(request, 'not a real context!')
     self.assertEqual(expected_response, response)
 
   def testUnaryCallFuture(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = request_pb2.SimpleRequest(response_size=13)
-      request = self._request_pb2.SimpleRequest(response_size=13)
       # Check that the call does not block waiting for the server to respond.
       with methods.pause():
         response_future = stub.UnaryCall.future(
@@ -348,9 +284,8 @@ class PythonPluginTest(unittest.TestCase):
     self.assertEqual(expected_response, response)
 
   def testUnaryCallFutureExpired(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = request_pb2.SimpleRequest(response_size=13)
-      request = self._request_pb2.SimpleRequest(response_size=13)
       with methods.pause():
         response_future = stub.UnaryCall.future(
             request, test_constants.SHORT_TIMEOUT)
@@ -358,27 +293,24 @@ class PythonPluginTest(unittest.TestCase):
           response_future.result()
 
   def testUnaryCallFutureCancelled(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = request_pb2.SimpleRequest(response_size=13)
-      request = self._request_pb2.SimpleRequest(response_size=13)
       with methods.pause():
         response_future = stub.UnaryCall.future(request, 1)
         response_future.cancel()
         self.assertTrue(response_future.cancelled())
 
   def testUnaryCallFutureFailed(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = request_pb2.SimpleRequest(response_size=13)
-      request = self._request_pb2.SimpleRequest(response_size=13)
       with methods.fail():
         response_future = stub.UnaryCall.future(
             request, test_constants.LONG_TIMEOUT)
         self.assertIsNotNone(response_future.exception())
 
   def testStreamingOutputCall(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request()
-      request = _streaming_output_request(self._request_pb2)
       responses = stub.StreamingOutputCall(
           request, test_constants.LONG_TIMEOUT)
       expected_responses = methods.StreamingOutputCall(
@@ -388,9 +320,8 @@ class PythonPluginTest(unittest.TestCase):
         self.assertEqual(expected_response, response)
 
   def testStreamingOutputCallExpired(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request()
-      request = _streaming_output_request(self._request_pb2)
       with methods.pause():
         responses = stub.StreamingOutputCall(
             request, test_constants.SHORT_TIMEOUT)
@@ -398,9 +329,8 @@ class PythonPluginTest(unittest.TestCase):
           list(responses)
 
   def testStreamingOutputCallCancelled(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request()
-      request = _streaming_output_request(self._request_pb2)
       responses = stub.StreamingOutputCall(
           request, test_constants.LONG_TIMEOUT)
       next(responses)
@@ -409,9 +339,8 @@ class PythonPluginTest(unittest.TestCase):
         next(responses)
 
   def testStreamingOutputCallFailed(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request = _streaming_output_request()
-      request = _streaming_output_request(self._request_pb2)
       with methods.fail():
         responses = stub.StreamingOutputCall(request, 1)
         self.assertIsNotNone(responses)
@@ -419,38 +348,32 @@ class PythonPluginTest(unittest.TestCase):
           next(responses)
 
   def testStreamingInputCall(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       response = stub.StreamingInputCall(
-          _streaming_input_request_iterator(
+          _streaming_input_request_iterator(),
-              self._request_pb2, self._payload_pb2),
           test_constants.LONG_TIMEOUT)
     expected_response = methods.StreamingInputCall(
-        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        _streaming_input_request_iterator(),
         'not a real RpcContext!')
     self.assertEqual(expected_response, response)
 
   def testStreamingInputCallFuture(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(
+            _streaming_input_request_iterator(),
-                self._request_pb2, self._payload_pb2),
             test_constants.LONG_TIMEOUT)
       response = response_future.result()
     expected_response = methods.StreamingInputCall(
-        _streaming_input_request_iterator(self._request_pb2, self._payload_pb2),
+        _streaming_input_request_iterator(),
         'not a real RpcContext!')
     self.assertEqual(expected_response, response)
 
   def testStreamingInputCallFutureExpired(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(
+            _streaming_input_request_iterator(),
-                self._request_pb2, self._payload_pb2),
             test_constants.SHORT_TIMEOUT)
         with self.assertRaises(face.ExpirationError):
           response_future.result()
@@ -458,12 +381,10 @@ class PythonPluginTest(unittest.TestCase):
             response_future.exception(), face.ExpirationError)
 
   def testStreamingInputCallFutureCancelled(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(
+            _streaming_input_request_iterator(),
-                self._request_pb2, self._payload_pb2),
             test_constants.LONG_TIMEOUT)
         response_future.cancel()
         self.assertTrue(response_future.cancelled())
@@ -471,32 +392,28 @@ class PythonPluginTest(unittest.TestCase):
         response_future.result()
 
   def testStreamingInputCallFutureFailed(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with methods.fail():
         response_future = stub.StreamingInputCall.future(
-            _streaming_input_request_iterator(
+            _streaming_input_request_iterator(),
-                self._request_pb2, self._payload_pb2),
             test_constants.LONG_TIMEOUT)
         self.assertIsNotNone(response_future.exception())
 
   def testFullDuplexCall(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       responses = stub.FullDuplexCall(
-          _full_duplex_request_iterator(self._request_pb2),
+          _full_duplex_request_iterator(),
           test_constants.LONG_TIMEOUT)
       expected_responses = methods.FullDuplexCall(
-          _full_duplex_request_iterator(self._request_pb2),
+          _full_duplex_request_iterator(),
           'not a real RpcContext!')
       for expected_response, response in moves.zip_longest(
           expected_responses, responses):
         self.assertEqual(expected_response, response)
 
   def testFullDuplexCallExpired(self):
-    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    request_iterator = _full_duplex_request_iterator()
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with methods.pause():
         responses = stub.FullDuplexCall(
             request_iterator, test_constants.SHORT_TIMEOUT)
@@ -504,9 +421,8 @@ class PythonPluginTest(unittest.TestCase):
           list(responses)
 
   def testFullDuplexCallCancelled(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
+      request_iterator = _full_duplex_request_iterator()
-      request_iterator = _full_duplex_request_iterator(self._request_pb2)
       responses = stub.FullDuplexCall(
           request_iterator, test_constants.LONG_TIMEOUT)
       next(responses)
@@ -515,9 +431,8 @@ class PythonPluginTest(unittest.TestCase):
         next(responses)
 
   def testFullDuplexCallFailed(self):
-    request_iterator = _full_duplex_request_iterator(self._request_pb2)
+    request_iterator = _full_duplex_request_iterator()
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with methods.fail():
         responses = stub.FullDuplexCall(
             request_iterator, test_constants.LONG_TIMEOUT)
@@ -526,13 +441,12 @@ class PythonPluginTest(unittest.TestCase):
           next(responses)
 
   def testHalfDuplexCall(self):
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       def half_duplex_request_iterator():
-        request = self._request_pb2.StreamingOutputCallRequest()
+        request = request_pb2.StreamingOutputCallRequest()
         request.response_parameters.add(size=1, interval_us=0)
         yield request
-        request = self._request_pb2.StreamingOutputCallRequest()
+        request = request_pb2.StreamingOutputCallRequest()
         request.response_parameters.add(size=2, interval_us=0)
         request.response_parameters.add(size=3, interval_us=0)
         yield request
@@ -557,14 +471,13 @@ class PythonPluginTest(unittest.TestCase):
         wait_cell[0] = False
         condition.notify_all()
     def half_duplex_request_iterator():
-      request = self._request_pb2.StreamingOutputCallRequest()
+      request = request_pb2.StreamingOutputCallRequest()
       request.response_parameters.add(size=1, interval_us=0)
       yield request
       with condition:
         while wait_cell[0]:
           condition.wait()
-    with _CreateService(self._service_pb2, self._response_pb2,
+    with _CreateService() as (methods, stub):
-                        self._payload_pb2) as (methods, stub):
       with wait():
         responses = stub.HalfDuplexCall(
             half_duplex_request_iterator(), test_constants.SHORT_TIMEOUT)
@@ -574,5 +487,4 @@ class PythonPluginTest(unittest.TestCase):
 
 
 if __name__ == '__main__':
-  #os.chdir(os.path.dirname(sys.argv[0]))
   unittest.main(verbosity=2)

src/python/grpcio_tests/tests/protoc_plugin/protos/requests/r/test_requests.proto

@@ -29,7 +29,7 @@
 
 syntax = "proto3";
 
-import "protos/payload/test_payload.proto";
+import "tests/protoc_plugin/protos/payload/test_payload.proto";
 
 package grpc_protoc_plugin;
 

src/python/grpcio_tests/tests/protoc_plugin/protos/responses/test_responses.proto

@@ -29,7 +29,7 @@
 
 syntax = "proto3";
 
-import "protos/payload/test_payload.proto";
+import "tests/protoc_plugin/protos/payload/test_payload.proto";
 
 package grpc_protoc_plugin;
 

src/python/grpcio_tests/tests/protoc_plugin/protos/service/test_service.proto

@@ -29,8 +29,8 @@
 
 syntax = "proto3";
 
-import "protos/requests/r/test_requests.proto";
+import "tests/protoc_plugin/protos/requests/r/test_requests.proto";
-import "protos/responses/test_responses.proto";
+import "tests/protoc_plugin/protos/responses/test_responses.proto";
 
 package grpc_protoc_plugin;
 

src/python/grpcio_tests/tests/unit/_sanity/_sanity_test.py

@@ -30,6 +30,9 @@
 import json
 import unittest
 
+import pkg_resources
+import six
+
 import tests
 
 
@@ -44,8 +47,10 @@ class Sanity(unittest.TestCase):
         for test_case_class in tests._loader.iterate_suite_cases(loader.suite)]
     test_suite_names = sorted(set(test_suite_names))
 
-    with open('src/python/grpcio/tests/tests.json') as tests_json_file:
+    tests_json_string = pkg_resources.resource_string('tests', 'tests.json')
-      tests_json = json.load(tests_json_file)
+    if six.PY3:
+      tests_json_string = tests_json_string.decode()
+    tests_json = json.loads(tests_json_string)
     self.assertListEqual(test_suite_names, tests_json)