Use precompiled extensions only

PYTHON-MANIFEST.in

@@ -7,6 +7,7 @@ graft third_party/zlib
 include src/python/grpcio/commands.py
 include src/python/grpcio/grpc_version.py
 include src/python/grpcio/grpc_core_dependencies.py
+include src/python/grpcio/precompiled.py
 include src/python/grpcio/support.py
 include src/python/grpcio/README.rst
 include requirements.txt

setup.py

@@ -53,6 +53,7 @@ sys.path.insert(0, os.path.abspath(PYTHON_STEM))
 
 # Break import-style to ensure we can actually find our in-repo dependencies.
 import commands
+import precompiled
 import grpc_core_dependencies
 import grpc_version
 
@@ -156,15 +157,14 @@ SETUP_REQUIRES = (
 ) + INSTALL_REQUIRES
 
 COMMAND_CLASS = {
-    'install': commands.Install,
     'doc': commands.SphinxDocumentation,
     'build_proto_modules': commands.BuildProtoModules,
     'build_project_metadata': commands.BuildProjectMetadata,
     'build_py': commands.BuildPy,
     'build_ext': commands.BuildExt,
+    'build_tagged_ext': precompiled.BuildTaggedExt,
     'gather': commands.Gather,
     'run_interop': commands.RunInterop,
-    'bdist_wheel_grpc_custom': commands.BdistWheelCustomName,
 }
 
 # Ensure that package data is copied over before any commands have been run:
@@ -205,9 +205,12 @@ PACKAGE_DATA = {
     'grpc._adapter': [
         'credentials/roots.pem'
     ],
+    # Binaries that may or may not be present in the final installation, but are
+    # mentioned here for completeness.
     'grpc._cython': [
         '_windows/grpc_c.32.python',
         '_windows/grpc_c.64.python',
+        'cygrpc.so',
     ],
 }
 if INSTALL_TESTS:
@@ -217,19 +220,22 @@ else:
   PACKAGES = setuptools.find_packages(
       PYTHON_STEM, exclude=['tests', 'tests.*'])
 
-setuptools.setup(
+setup_arguments = {
-    name='grpcio',
+    'name': 'grpcio',
-    version=grpc_version.VERSION,
+    'version': grpc_version.VERSION,
-    license=LICENSE,
+    'license': LICENSE,
-    ext_modules=CYTHON_EXTENSION_MODULES,
+    'ext_modules': CYTHON_EXTENSION_MODULES,
-    packages=list(PACKAGES),
+    'packages': list(PACKAGES),
-    package_dir=PACKAGE_DIRECTORIES,
+    'package_dir': PACKAGE_DIRECTORIES,
-    package_data=PACKAGE_DATA,
+    'package_data': PACKAGE_DATA,
-    install_requires=INSTALL_REQUIRES,
+    'install_requires': INSTALL_REQUIRES,
-    setup_requires=SETUP_REQUIRES,
+    'setup_requires': SETUP_REQUIRES,
-    cmdclass=COMMAND_CLASS,
+    'cmdclass': COMMAND_CLASS,
-    tests_require=TESTS_REQUIRE,
+    'tests_require': TESTS_REQUIRE,
-    test_suite=TEST_SUITE,
+    'test_loader': TEST_LOADER,
-    test_loader=TEST_LOADER,
+    'test_runner': TEST_RUNNER,
-    test_runner=TEST_RUNNER,
+}
-)
+
+precompiled.update_setup_arguments(setup_arguments)
+
+setuptools.setup(**setup_arguments)

src/python/grpcio/commands.py

@@ -46,21 +46,11 @@ from setuptools.command import build_py
 from setuptools.command import easy_install
 from setuptools.command import install
 from setuptools.command import test
-from wheel import bdist_wheel
 
 import support
 
 PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
 
-BINARIES_REPOSITORY = os.environ.get(
-    'GRPC_PYTHON_BINARIES_REPOSITORY',
-    'https://storage.googleapis.com/grpc-precompiled-binaries/python')
-
-USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get(
-    'GRPC_PYTHON_USE_CUSTOM_BDIST', '1')))
-
-GRPC_CUSTOM_BDIST_EXT = '.whl'
-
 CONF_PY_ADDENDUM = """
 extensions.append('sphinx.ext.napoleon')
 napoleon_google_docstring = True
@@ -111,98 +101,6 @@ def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename):
   return bdist_path
 
 
-class WheelNameMixin(object):
-  """Mixin for setuptools.Command classes to enable acquiring the bdist name."""
-
-  def wheel_custom_name(self):
-    base = self.wheel_name()
-    # Drop troublesome parts of the target tuple
-    base_split = base.split('-')
-    base = '-'.join(base_split[0:3] + base_split[4:])
-    flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
-    return '{base}-{flavor}'.format(base=base, flavor=flavor)
-
-  def wheel_name(self):
-    wheel_command = self.get_finalized_command('bdist_wheel')
-    return wheel_command.get_archive_basename()
-
-
-class Install(install.install, WheelNameMixin):
-  """Custom Install command for gRPC Python.
-
-  This is for bdist shims and whatever else we might need a custom install
-  command for.
-  """
-
-  user_options = install.install.user_options + [
-      # TODO(atash): remove this once PyPI has better Linux bdist support. See
-      # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-      ('use-grpc-custom-bdist', None,
-       'Whether to retrieve a binary from the gRPC binary repository instead '
-       'of building from source.'),
-  ]
-
-  def initialize_options(self):
-    install.install.initialize_options(self)
-    self.use_grpc_custom_bdist = USE_GRPC_CUSTOM_BDIST
-
-  def finalize_options(self):
-    install.install.finalize_options(self)
-
-  def run(self):
-    if self.use_grpc_custom_bdist:
-      try:
-        try:
-          bdist_path = _get_grpc_custom_bdist(self.wheel_custom_name(),
-                                              self.wheel_name())
-        except CommandError as error:
-          sys.stderr.write(
-              '\nWARNING: Failed to acquire grpcio prebuilt binary:\n'
-              '{}.\n\n'.format(error.message))
-          raise
-        try:
-          self._run_bdist_retrieval_install(bdist_path)
-        except Exception as error:
-          # if anything else happens (and given how there's no way to really know
-          # what's happening in setuptools here, I mean *anything*), warn the user
-          # and fall back to building from source.
-          sys.stderr.write(
-              '{}\nWARNING: Failed to install grpcio prebuilt binary.\n\n'
-                  .format(traceback.format_exc()))
-          raise
-      except Exception:
-        install.install.run(self)
-    else:
-      install.install.run(self)
-
-  # TODO(atash): Remove this once PyPI has better Linux bdist support. See
-  # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-  def _run_bdist_retrieval_install(self, bdist_path):
-    import pip
-    pip.main(['install', bdist_path])
-
-
-class BdistWheelCustomName(bdist_wheel.bdist_wheel, WheelNameMixin):
-  """Thin wrapper around the bdist command to build with our custom name."""
-
-  description = ("Create a gRPC custom-named wheel distribution. "
-                 "Cannot be run with any other distribution-related command.")
-
-  def run(self):
-    # TODO(atash): if the hack we use to support Linux binaries becomes
-    # 'supported' (i.e.
-    # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-    # is not solved and we see users beginning to use this command, ill-advised
-    # as that may be) consider making the following capable of running with
-    # other distribution-related commands. Currently it depends on the (AFAIK
-    # undocumented, private) ordering of the distribution files.
-    bdist_wheel.bdist_wheel.run(self)
-    output = self.distribution.dist_files[-1][2]
-    target = os.path.join(
-        self.dist_dir, '{}.whl'.format(self.wheel_custom_name()))
-    shutil.move(output, target)
-
-
 class SphinxDocumentation(setuptools.Command):
   """Command to generate documentation via sphinx."""
 

src/python/grpcio/precompiled.py

@@ -0,0 +1,102 @@
+# Copyright 2015-2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import os
+import platform
+import shutil
+import sys
+
+import setuptools
+
+import commands
+import grpc_version
+
+try:
+  from urllib2 import urlopen
+except ImportError:
+  from urllib.request import urlopen
+
+PYTHON_STEM = os.path.dirname(os.path.abspath(__file__))
+BINARIES_REPOSITORY = os.environ.get(
+    'GRPC_PYTHON_BINARIES_REPOSITORY',
+    'https://storage.googleapis.com/grpc-precompiled-binaries/python')
+USE_PRECOMPILED_BINARIES = bool(int(os.environ.get(
+    'GRPC_PYTHON_USE_PRECOMPILED_BINARIES', '1')))
+
+def _tagged_ext_name(base):
+  uname = platform.uname()
+  tags = '-'.join((grpc_version.VERSION, uname[0], uname[4]))
+  flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
+  return '{base}-{tags}-{flavor}'.format(base=base, tags=tags, flavor=flavor)
+
+
+class BuildTaggedExt(setuptools.Command):
+
+  description = 'build the gRPC tagged extensions'
+  user_options = []
+
+  def initialize_options(self):
+    # distutils requires this override.
+    pass
+
+  def finalize_options(self):
+    # distutils requires this override.
+    pass
+
+  def run(self):
+    if 'linux' in sys.platform:
+      self.run_command('build_ext')
+      try:
+        os.makedirs('dist/')
+      except OSError:
+        pass
+      shutil.copyfile(
+          os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so'),
+          'dist/{}.so'.format(_tagged_ext_name('cygrpc')))
+    else:
+      sys.stderr.write('nothing to do for build_tagged_ext\n')
+
+
+def update_setup_arguments(setup_arguments):
+  url = '{}/{}.so'.format(BINARIES_REPOSITORY, _tagged_ext_name('cygrpc'))
+  target_path = os.path.join(PYTHON_STEM, 'grpc/_cython/cygrpc.so')
+  try:
+    extension = urlopen(url).read()
+  except:
+    sys.stderr.write(
+        'could not download precompiled extension: {}\n'.format(url))
+    return
+  try:
+    with open(target_path, 'w') as target:
+      target.write(extension)
+    setup_arguments['ext_modules'] = []
+  except:
+    sys.stderr.write(
+        'could not write precompiled extension to directory: {} -> {}\n'
+            .format(url, target_path))

tools/run_tests/build_artifact_python.sh

@@ -46,7 +46,7 @@ fi
 GRPC_PYTHON_USE_CUSTOM_BDIST=0  \
 GRPC_PYTHON_BUILD_WITH_CYTHON=1 \
 ${SETARCH_CMD} python setup.py  \
-    bdist_wheel_grpc_custom
+    build_tagged_ext
 
 GRPC_PYTHON_USE_CUSTOM_BDIST=0  \
 GRPC_PYTHON_BUILD_WITH_CYTHON=1 \