From 9d1ba2efd9ae4db1833eb9b58a76bbc5eaebac72 Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Mon, 13 Nov 2017 14:16:21 -0800 Subject: [PATCH] Fix grpcio_{health_checking,reflection} packaging The previous packaging structure exhibited strange behavior of slowness when trying to use pip to install grpcio-reflection or grpcio-health-checking in a single line with grpcio-tools. The root cause seems to be the complicated interaction between pip and setuptools and the fact that we ship a single .tar.gz "source" archive for `grpcio_reflection` and `grpcio_health_checking` packages. `pip` tries to build this "source" package, and our build process wants to generate code for the `.proto` files in the package. However, we have already processed the `.proto` files into `_pb2.py` files in our artifact build process, and installing `grpcio_tools` to get `grpcio_{reflection,health_checking}` seems excessive. The behavior gets worse since `setuptools`, while building the package from source, tries to fetch `grpcio_tools` from source and build that too. This takes a while, since it involves compiling a bunch of native code from `protobuf` and `grpc` and requires a C compiler to boot. This commit modifies the Python artifact for the two packages so that they will not include the raw `.proto` files in the distribution uploaded to PyPI, nor would they contain the Python module that does the preprocessing code generation from the respective .proto files. Instead, a specific code path is taken when the generated `_pb2_grpc` Python module is not present in the package to provide such functionality when built from the gRPC git repository (and hence when built from our CI infrastructure.) --- src/python/grpcio_health_checking/MANIFEST.in | 3 +- src/python/grpcio_health_checking/setup.py | 47 +++++++++++++++---- src/python/grpcio_reflection/MANIFEST.in | 3 +- src/python/grpcio_reflection/setup.py | 47 +++++++++++++++---- 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/src/python/grpcio_health_checking/MANIFEST.in b/src/python/grpcio_health_checking/MANIFEST.in index 5255e4c4036..996c74a9d49 100644 --- a/src/python/grpcio_health_checking/MANIFEST.in +++ b/src/python/grpcio_health_checking/MANIFEST.in @@ -1,4 +1,3 @@ include grpc_version.py -include health_commands.py -graft grpc_health +recursive-include grpc_health *.py global-exclude *.pyc diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 1f5e9c5130b..01d796f4e6c 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -20,10 +20,26 @@ import setuptools # Ensure we're in the proper directory whether or not we're being used by pip. os.chdir(os.path.dirname(os.path.abspath(__file__))) -# Break import-style to ensure we can actually find our commands module. -import health_commands +# Break import-style to ensure we can actually find our local modules. import grpc_version + +class _NoOpCommand(setuptools.Command): + """No-op command.""" + + description = '' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + pass + + CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', @@ -40,17 +56,28 @@ PACKAGE_DIRECTORIES = { '': '.', } -SETUP_REQUIRES = ( - 'grpcio-tools>={version}'.format(version=grpc_version.VERSION),) - INSTALL_REQUIRES = ('protobuf>=3.3.0', 'grpcio>={version}'.format(version=grpc_version.VERSION),) -COMMAND_CLASS = { - # Run preprocess from the repository *before* doing any packaging! - 'preprocess': health_commands.CopyProtoModules, - 'build_package_protos': health_commands.BuildPackageProtos, -} +try: + # ensure we can load the _pb2_grpc module: + from grpc_health.v1 import health_pb2_grpc as _pb2_grpc + # if we can find the _pb2_grpc module, the package has already been built. + SETUP_REQUIRES = () + COMMAND_CLASS = { + # wire up commands to no-op not to break the external dependencies + 'preprocess': _NoOpCommand, + 'build_package_protos': _NoOpCommand, + } +except ImportError: # we are in the build environment + import health_commands as _health_commands + SETUP_REQUIRES = ( + 'grpcio-tools=={version}'.format(version=grpc_version.VERSION),) + COMMAND_CLASS = { + # Run preprocess from the repository *before* doing any packaging! + 'preprocess': _health_commands.CopyProtoModules, + 'build_package_protos': _health_commands.BuildPackageProtos, + } setuptools.setup( name='grpcio-health-checking', diff --git a/src/python/grpcio_reflection/MANIFEST.in b/src/python/grpcio_reflection/MANIFEST.in index 0f2130c0b58..d6fb6ce73aa 100644 --- a/src/python/grpcio_reflection/MANIFEST.in +++ b/src/python/grpcio_reflection/MANIFEST.in @@ -1,4 +1,3 @@ include grpc_version.py -include reflection_commands.py -graft grpc_reflection +recursive-include grpc_reflection *.py global-exclude *.pyc diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 9360550afbc..ad9e86990fc 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -21,10 +21,26 @@ import setuptools # Ensure we're in the proper directory whether or not we're being used by pip. os.chdir(os.path.dirname(os.path.abspath(__file__))) -# Break import-style to ensure we can actually find our commands module. -import reflection_commands +# Break import-style to ensure we can actually find our local modules. import grpc_version + +class _NoOpCommand(setuptools.Command): + """No-op command.""" + + description = '' + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + pass + + CLASSIFIERS = [ 'Development Status :: 5 - Production/Stable', 'Programming Language :: Python', @@ -41,17 +57,28 @@ PACKAGE_DIRECTORIES = { '': '.', } -SETUP_REQUIRES = ( - 'grpcio-tools>={version}'.format(version=grpc_version.VERSION),) - INSTALL_REQUIRES = ('protobuf>=3.3.0', 'grpcio>={version}'.format(version=grpc_version.VERSION),) -COMMAND_CLASS = { - # Run preprocess from the repository *before* doing any packaging! - 'preprocess': reflection_commands.CopyProtoModules, - 'build_package_protos': reflection_commands.BuildPackageProtos, -} +try: + # ensure we can load the _pb2_grpc module: + from grpc_reflection.v1alpha import reflection_pb2_grpc as _pb2_grpc + # if we can find the _pb2_grpc module, the package has already been built. + SETUP_REQUIRES = () + COMMAND_CLASS = { + # wire up commands to no-op not to break the external dependencies + 'preprocess': _NoOpCommand, + 'build_package_protos': _NoOpCommand, + } +except ImportError: # we are in the build environment + import reflection_commands as _reflection_commands + SETUP_REQUIRES = ( + 'grpcio-tools=={version}'.format(version=grpc_version.VERSION),) + COMMAND_CLASS = { + # Run preprocess from the repository *before* doing any packaging! + 'preprocess': _reflection_commands.CopyProtoModules, + 'build_package_protos': _reflection_commands.BuildPackageProtos, + } setuptools.setup( name='grpcio-reflection',