Merge pull request #5194 from soltanmm/everything-is-awful!

Use wheels instead of eggs for custom bdists.

requirements.txt

@@ -4,3 +4,4 @@ futures>=2.2.0
 cython>=0.23
 coverage>=4.0
 six>=1.10
+wheel>=0.29

setup.py

@@ -164,7 +164,7 @@ COMMAND_CLASS = {
     'build_ext': commands.BuildExt,
     'gather': commands.Gather,
     'run_interop': commands.RunInterop,
-    'bdist_egg_grpc_custom': commands.BdistEggCustomName,
+    'bdist_wheel_grpc_custom': commands.BdistWheelCustomName,
 }
 
 # Ensure that package data is copied over before any commands have been run:

src/python/grpcio/commands.py

@@ -41,12 +41,12 @@ import sys
 import traceback
 
 import setuptools
-from setuptools.command import bdist_egg
 from setuptools.command import build_ext
 from setuptools.command import build_py
 from setuptools.command import easy_install
 from setuptools.command import install
 from setuptools.command import test
+from wheel import bdist_wheel
 
 import support
 
@@ -59,6 +59,8 @@ BINARIES_REPOSITORY = os.environ.get(
 USE_GRPC_CUSTOM_BDIST = bool(int(os.environ.get(
     'GRPC_PYTHON_USE_CUSTOM_BDIST', '1')))
 
+GRPC_CUSTOM_BDIST_EXT = '.whl'
+
 CONF_PY_ADDENDUM = """
 extensions.append('sphinx.ext.napoleon')
 napoleon_google_docstring = True
@@ -74,46 +76,52 @@ class CommandError(Exception):
 
 # TODO(atash): Remove this once PyPI has better Linux bdist support. See
 # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-def _get_grpc_custom_bdist_egg(decorated_basename, target_egg_basename):
+def _get_grpc_custom_bdist(decorated_basename, target_bdist_basename):
-  """Returns a string path to a .egg file for Linux to install.
+  """Returns a string path to a bdist file for Linux to install.
 
-  If we can retrieve a pre-compiled egg from online, uses it. Else, emits a
+  If we can retrieve a pre-compiled bdist from online, uses it. Else, emits a
   warning and builds from source.
   """
+  # TODO(atash): somehow the name that's returned from `wheel` is different
+  # between different versions of 'wheel' (but from a compatibility standpoint,
+  # the names are compatible); we should have some way of determining name
+  # compatibility in the same way `wheel` does to avoid having to rename all of
+  # the custom wheels that we build/upload to GCS.
+
   # Break import style to ensure that setup.py has had a chance to install the
-  # relevant package eggs.
+  # relevant package.
   from six.moves.urllib import request
-  decorated_path = decorated_basename + '.egg'
+  decorated_path = decorated_basename + GRPC_CUSTOM_BDIST_EXT
   try:
     url = BINARIES_REPOSITORY + '/{target}'.format(target=decorated_path)
-    egg_data = request.urlopen(url).read()
+    bdist_data = request.urlopen(url).read()
   except IOError as error:
     raise CommandError(
-        '{}\n\nCould not find the bdist egg {}: {}'
+        '{}\n\nCould not find the bdist {}: {}'
             .format(traceback.format_exc(), decorated_path, error.message))
-  # Our chosen local egg path.
+  # Our chosen local bdist path.
-  egg_path = target_egg_basename + '.egg'
+  bdist_path = target_bdist_basename + GRPC_CUSTOM_BDIST_EXT
   try:
-    with open(egg_path, 'w') as egg_file:
+    with open(bdist_path, 'w') as bdist_file:
-      egg_file.write(egg_data)
+      bdist_file.write(bdist_data)
   except IOError as error:
     raise CommandError(
-        '{}\n\nCould not write grpcio egg: {}'
+        '{}\n\nCould not write grpcio bdist: {}'
             .format(traceback.format_exc(), error.message))
-  return egg_path
+  return bdist_path
 
 
-class EggNameMixin(object):
+class WheelNameMixin(object):
-  """Mixin for setuptools.Command classes to enable acquiring the egg name."""
+  """Mixin for setuptools.Command classes to enable acquiring the bdist name."""
 
-  def egg_name(self, with_custom):
+  def wheel_name(self, with_custom):
     """
     Args:
-      with_custom: Boolean describing whether or not to decorate the egg name
+      with_custom: Boolean describing whether or not to decorate the bdist name
         with custom gRPC-specific target information.
     """
-    egg_command = self.get_finalized_command('bdist_egg')
+    wheel_command = self.get_finalized_command('bdist_wheel')
-    base = os.path.splitext(os.path.basename(egg_command.egg_output))[0]
+    base = wheel_command.get_archive_basename()
     if with_custom:
       flavor = 'ucs2' if sys.maxunicode == 65535 else 'ucs4'
       return '{base}-{flavor}'.format(base=base, flavor=flavor)
@@ -121,7 +129,7 @@ class EggNameMixin(object):
       return base
 
 
-class Install(install.install, EggNameMixin):
+class Install(install.install, WheelNameMixin):
   """Custom Install command for gRPC Python.
 
   This is for bdist shims and whatever else we might need a custom install
@@ -147,15 +155,15 @@ class Install(install.install, EggNameMixin):
     if self.use_grpc_custom_bdist:
       try:
         try:
-          egg_path = _get_grpc_custom_bdist_egg(self.egg_name(True),
+          bdist_path = _get_grpc_custom_bdist(self.wheel_name(True),
-                                                self.egg_name(False))
+                                            self.wheel_name(False))
         except CommandError as error:
           sys.stderr.write(
               '\nWARNING: Failed to acquire grpcio prebuilt binary:\n'
               '{}.\n\n'.format(error.message))
           raise
         try:
-          self._run_bdist_retrieval_install(egg_path)
+          self._run_bdist_retrieval_install(bdist_path)
         except Exception as error:
           # if anything else happens (and given how there's no way to really know
           # what's happening in setuptools here, I mean *anything*), warn the user
@@ -171,29 +179,19 @@ class Install(install.install, EggNameMixin):
 
   # TODO(atash): Remove this once PyPI has better Linux bdist support. See
   # https://bitbucket.org/pypa/pypi/issues/120/binary-wheels-for-linux-are-not-supported
-  def _run_bdist_retrieval_install(self, bdist_egg):
+  def _run_bdist_retrieval_install(self, bdist_path):
-    easy_install = self.distribution.get_command_class('easy_install')
+    import pip
-    easy_install_command = easy_install(
+    pip.main(['install', bdist_path])
-        self.distribution, args='x', root=self.root, record=self.record,
+
-    )
+
-    easy_install_command.ensure_finalized()
+class BdistWheelCustomName(bdist_wheel.bdist_wheel, WheelNameMixin):
-    easy_install_command.always_copy_from = '.'
+  """Thin wrapper around the bdist command to build with our custom name."""
-    easy_install_command.package_index.scan(glob.glob('*.egg'))
-    arguments = [bdist_egg]
-    if setuptools.bootstrap_install_from:
-      args.insert(0, setuptools.bootstrap_install_from)
-    easy_install_command.args = arguments
-    easy_install_command.run()
-    setuptools.bootstrap_install_from = None
-
-
-class BdistEggCustomName(bdist_egg.bdist_egg, EggNameMixin):
-  """Thin wrapper around the bdist_egg command to build with our custom name."""
 
   def run(self):
-    bdist_egg.bdist_egg.run(self)
+    bdist_wheel.bdist_wheel.run(self)
-    target = os.path.join(self.dist_dir, '{}.egg'.format(self.egg_name(True)))
+    output = self.distribution.dist_files[-1][2]
-    shutil.move(self.get_outputs()[0], target)
+    target = os.path.join(self.dist_dir, '{}.whl'.format(self.wheel_name(True)))
+    shutil.move(output, target)
 
 
 class SphinxDocumentation(setuptools.Command):

tools/run_tests/build_artifact_python.sh

@@ -44,7 +44,7 @@ GRPC_PYTHON_BUILD_WITH_CYTHON=1 \
 ${SETARCH_CMD} python setup.py  \
     bdist_wheel                 \
     sdist                       \
-    bdist_egg_grpc_custom
+    bdist_wheel_grpc_custom
 
 mkdir -p artifacts