From 08e117ae806225fd704ab4221c1014b5fa495482 Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 17 Jun 2020 16:12:59 -0700 Subject: [PATCH 1/6] Improve build slightly --- setup.py | 1 + src/python/grpcio/_parallel_compile_patch.py | 5 ++- src/python/grpcio/commands.py | 45 +++++++++++++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 807d005899d..3f27ebd4c14 100644 --- a/setup.py +++ b/setup.py @@ -383,6 +383,7 @@ COMMAND_CLASS = { 'build_py': commands.BuildPy, 'build_ext': commands.BuildExt, 'gather': commands.Gather, + 'clean': commands.Clean, } # Ensure that package data is copied over before any commands have been run: diff --git a/src/python/grpcio/_parallel_compile_patch.py b/src/python/grpcio/_parallel_compile_patch.py index b34aa17fd0b..e4d50c38311 100644 --- a/src/python/grpcio/_parallel_compile_patch.py +++ b/src/python/grpcio/_parallel_compile_patch.py @@ -22,7 +22,10 @@ import os try: BUILD_EXT_COMPILER_JOBS = int( - os.environ.get('GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS', '1')) + os.environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS']) +except KeyError: + import multiprocessing + BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count() except ValueError: BUILD_EXT_COMPILER_JOBS = 1 diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index b9106826b5d..6d7a286632d 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -13,6 +13,8 @@ # limitations under the License. """Provides distutils command classes for the GRPC Python setup process.""" +from __future__ import print_function + import distutils import glob import os @@ -218,7 +220,10 @@ class BuildExt(build_ext.build_ext): if platform.system() != 'Windows': return False # TODO(lidiz) Remove the generated a.out for success tests. - cc_test = subprocess.Popen(['cc', '-x', 'c', '-std=c++11', '-'], + cc_test = subprocess.Popen([ + distutils.ccompiler.executable_filename, '-x', 'c', + '-std=c++11', '-' + ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -290,3 +295,41 @@ class Gather(setuptools.Command): self.distribution.install_requires) if self.test and self.distribution.tests_require: self.distribution.fetch_build_eggs(self.distribution.tests_require) + + +class Clean(setuptools.Command): + """Command to clean build artifacts.""" + + description = 'Clean build artifacts.' + user_options = [] + + _FILE_PATTERNS = [ + 'python_build', + 'src/python/grpcio/__pycache__/', + 'src/python/grpcio/grpc/_cython/cygrpc.cpp', + 'src/python/grpcio/grpc/_cython/*.so', + 'src/python/grpcio/grpcio.egg-info/', + ] + _CURRENT_DIRECTORY = os.path.normpath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + for path_spec in self._FILE_PATTERNS: + this_glob = os.path.normpath( + os.path.join(Clean._CURRENT_DIRECTORY, path_spec)) + abs_paths = glob.glob(this_glob) + for path in abs_paths: + if not str(path).startswith(Clean._CURRENT_DIRECTORY): + raise ValueError( + "Cowardly refusing to delete {}.".format(path)) + print("Removing {}".format(os.path.relpath(path))) + if os.path.isfile(path): + os.remove(str(path)) + else: + shutil.rmtree(str(path)) From f22e247183876cffbb3768eb73199fd31eb277fd Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 17 Jun 2020 22:00:15 -0700 Subject: [PATCH 2/6] Add script to install all Python modules --- tools/distrib/install_all_python_modules.sh | 25 +++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 tools/distrib/install_all_python_modules.sh diff --git a/tools/distrib/install_all_python_modules.sh b/tools/distrib/install_all_python_modules.sh new file mode 100755 index 00000000000..fd036cb9a5e --- /dev/null +++ b/tools/distrib/install_all_python_modules.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +echo "It's recommended that you run this script from a virtual environment." + +set -e + +BASEDIR=$(dirname "$0") +BASEDIR=$(realpath "$BASEDIR")/../.. + +(cd "$BASEDIR"; + pip install cython; + python setup.py install; + pushd tools/distrib/python/grpcio_tools; + ../make_grpcio_tools.py + GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install . + popd; + pushd src/python; + for PACKAGE in ./grpcio_*; do + pushd "${PACKAGE}"; + python setup.py preprocess; + python setup.py install; + popd; + done + popd; +) From edcf2100503e4aa658df1f57b37973c05cb4370b Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Wed, 17 Jun 2020 22:00:54 -0700 Subject: [PATCH 3/6] Use proper compiler --- src/python/grpcio/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 6d7a286632d..61332995393 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -221,7 +221,7 @@ class BuildExt(build_ext.build_ext): return False # TODO(lidiz) Remove the generated a.out for success tests. cc_test = subprocess.Popen([ - distutils.ccompiler.executable_filename, '-x', 'c', + self.compiler.compiler[0], '-x', 'c', '-std=c++11', '-' ], stdin=subprocess.PIPE, From af222241a7f7adb8ab49ea4a027c7464cab3e25e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Thu, 18 Jun 2020 10:40:31 -0700 Subject: [PATCH 4/6] Give up on getting compiler executable from distutils --- src/python/grpcio/commands.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 61332995393..dc441a2efd6 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -220,10 +220,7 @@ class BuildExt(build_ext.build_ext): if platform.system() != 'Windows': return False # TODO(lidiz) Remove the generated a.out for success tests. - cc_test = subprocess.Popen([ - self.compiler.compiler[0], '-x', 'c', - '-std=c++11', '-' - ], + cc_test = subprocess.Popen(['cc', '-x', 'c', '-std=c++11', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 98bb4b612448303eadd2282746303e08e6665fbb Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 19 Jun 2020 09:54:04 -0700 Subject: [PATCH 5/6] Copyright header --- tools/distrib/install_all_python_modules.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tools/distrib/install_all_python_modules.sh b/tools/distrib/install_all_python_modules.sh index fd036cb9a5e..51a11341bb6 100755 --- a/tools/distrib/install_all_python_modules.sh +++ b/tools/distrib/install_all_python_modules.sh @@ -1,4 +1,17 @@ #!/bin/bash +# Copyright 2020 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. echo "It's recommended that you run this script from a virtual environment." From 4fcf9d01d204bd03e9f4b364130a18796cc8fd6e Mon Sep 17 00:00:00 2001 From: Richard Belleville Date: Fri, 19 Jun 2020 15:31:55 -0700 Subject: [PATCH 6/6] Address review comments --- src/python/grpcio/commands.py | 4 ++-- tools/distrib/install_all_python_modules.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index dc441a2efd6..fb678eb2d53 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -300,13 +300,13 @@ class Clean(setuptools.Command): description = 'Clean build artifacts.' user_options = [] - _FILE_PATTERNS = [ + _FILE_PATTERNS = ( 'python_build', 'src/python/grpcio/__pycache__/', 'src/python/grpcio/grpc/_cython/cygrpc.cpp', 'src/python/grpcio/grpc/_cython/*.so', 'src/python/grpcio/grpcio.egg-info/', - ] + ) _CURRENT_DIRECTORY = os.path.normpath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../..")) diff --git a/tools/distrib/install_all_python_modules.sh b/tools/distrib/install_all_python_modules.sh index 51a11341bb6..8189a0ffc3b 100755 --- a/tools/distrib/install_all_python_modules.sh +++ b/tools/distrib/install_all_python_modules.sh @@ -21,7 +21,7 @@ BASEDIR=$(dirname "$0") BASEDIR=$(realpath "$BASEDIR")/../.. (cd "$BASEDIR"; - pip install cython; + pip install --upgrade cython; python setup.py install; pushd tools/distrib/python/grpcio_tools; ../make_grpcio_tools.py