diff --git a/BUILD b/BUILD index 423cb36a8aa..24c1fb31ced 100644 --- a/BUILD +++ b/BUILD @@ -63,6 +63,11 @@ config_setting( values = {"cpu": "x64_windows_msvc"}, ) +config_setting( + name = "python3", + values = {"python_path": "python3"}, +) + config_setting( name = "mac_x86_64", values = {"cpu": "darwin"}, diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel index 6958ccdfb66..27d5d2e4bb2 100644 --- a/src/python/grpcio/grpc/BUILD.bazel +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -15,9 +15,11 @@ py_library( "//src/python/grpcio/grpc/_cython:cygrpc", "//src/python/grpcio/grpc/experimental", "//src/python/grpcio/grpc/framework", - requirement('enum34'), requirement('six'), - ], + ] + select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), data = [ "//:grpc", ], diff --git a/src/python/grpcio/grpc/framework/common/BUILD.bazel b/src/python/grpcio/grpc/framework/common/BUILD.bazel index 9d9ef682c90..52fbb2b516c 100644 --- a/src/python/grpcio/grpc/framework/common/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/common/BUILD.bazel @@ -13,15 +13,17 @@ py_library( py_library( name = "cardinality", srcs = ["cardinality.py"], - deps = [ - requirement("enum34"), - ], + deps = select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), ) py_library( name = "style", srcs = ["style.py"], - deps = [ - requirement("enum34"), - ], + deps = select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), ) diff --git a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel index 1287fdd44ed..a447ecded49 100644 --- a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel @@ -23,9 +23,11 @@ py_library( name = "callable_util", srcs = ["callable_util.py"], deps = [ - requirement("enum34"), requirement("six"), - ], + ] + select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), ) py_library( @@ -39,9 +41,10 @@ py_library( py_library( name = "logging_pool", srcs = ["logging_pool.py"], - deps = [ - requirement("futures"), - ], + deps = select({ + "//conditions:default": [requirement('futures'),], + "//:python3": [], + }), ) py_library( diff --git a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel index 408a66a6310..35cfe877f34 100644 --- a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel @@ -15,15 +15,18 @@ py_library( srcs = ["base.py"], deps = [ "//src/python/grpcio/grpc/framework/foundation:abandonment", - requirement("enum34"), requirement("six"), - ], + ] + select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), ) py_library( name = "utilities", srcs = ["utilities.py"], - deps = [ - requirement("enum34"), - ], + deps = select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), ) diff --git a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel index e683e7cc426..83fadb6372e 100644 --- a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel +++ b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel @@ -16,9 +16,11 @@ py_library( deps = [ "//src/python/grpcio/grpc/framework/foundation", "//src/python/grpcio/grpc/framework/common", - requirement("enum34"), requirement("six"), - ], + ] + select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), ) py_library( diff --git a/src/python/grpcio_status/grpc_status/rpc_status.py b/src/python/grpcio_status/grpc_status/rpc_status.py index 87618fa5412..76891e2422e 100644 --- a/src/python/grpcio_status/grpc_status/rpc_status.py +++ b/src/python/grpcio_status/grpc_status/rpc_status.py @@ -17,11 +17,6 @@ import collections import grpc -# TODO(https://github.com/bazelbuild/bazel/issues/6844) -# Due to Bazel issue, the namespace packages won't resolve correctly. -# Adding this unused-import as a workaround to avoid module-not-found error -# under Bazel builds. -import google.protobuf # pylint: disable=unused-import from google.rpc import status_pb2 _CODE_TO_GRPC_CODE_MAPPING = {x.value[0]: x for x in grpc.StatusCode} diff --git a/src/python/grpcio_tests/tests/BUILD.bazel b/src/python/grpcio_tests/tests/BUILD.bazel new file mode 100644 index 00000000000..b908ab85173 --- /dev/null +++ b/src/python/grpcio_tests/tests/BUILD.bazel @@ -0,0 +1,8 @@ +py_library( + name = "bazel_namespace_package_hack", + srcs = ["bazel_namespace_package_hack.py"], + visibility = [ + "//src/python/grpcio_tests/tests/status:__subpackages__", + "//src/python/grpcio_tests/tests/interop:__subpackages__", + ], +) diff --git a/src/python/grpcio_tests/tests/bazel_namespace_package_hack.py b/src/python/grpcio_tests/tests/bazel_namespace_package_hack.py new file mode 100644 index 00000000000..c6b72c327b1 --- /dev/null +++ b/src/python/grpcio_tests/tests/bazel_namespace_package_hack.py @@ -0,0 +1,32 @@ +# Copyright 2019 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import site +import sys + + +# TODO(https://github.com/bazelbuild/bazel/issues/6844) Bazel failed to +# interpret namespace packages correctly. This monkey patch will force the +# Python process to parse the .pth file in the sys.path to resolve namespace +# package in the right place. +# Analysis in depth: https://github.com/bazelbuild/rules_python/issues/55 +def sys_path_to_site_dir_hack(): + """Add valid sys.path item to site directory to parse the .pth files.""" + for item in sys.path: + if os.path.exists(item): + # The only difference between sys.path and site-directory is + # whether the .pth file will be parsed or not. A site-directory + # will always exist in sys.path, but not another way around. + site.addsitedir(item) diff --git a/src/python/grpcio_tests/tests/interop/BUILD.bazel b/src/python/grpcio_tests/tests/interop/BUILD.bazel index aebdbf67ebf..770b1f78a70 100644 --- a/src/python/grpcio_tests/tests/interop/BUILD.bazel +++ b/src/python/grpcio_tests/tests/interop/BUILD.bazel @@ -29,17 +29,20 @@ py_library( srcs = ["methods.py"], deps = [ "//src/python/grpcio/grpc:grpcio", + "//src/python/grpcio_tests/tests:bazel_namespace_package_hack", "//src/proto/grpc/testing:py_empty_proto", "//src/proto/grpc/testing:py_messages_proto", "//src/proto/grpc/testing:py_test_proto", requirement('google-auth'), requirement('requests'), - requirement('enum34'), requirement('urllib3'), requirement('chardet'), requirement('certifi'), requirement('idna'), - ], + ] + select({ + "//conditions:default": [requirement('enum34'),], + "//:python3": [], + }), imports=["../../",], ) diff --git a/src/python/grpcio_tests/tests/interop/methods.py b/src/python/grpcio_tests/tests/interop/methods.py index c11f6c8fad7..40341ca091b 100644 --- a/src/python/grpcio_tests/tests/interop/methods.py +++ b/src/python/grpcio_tests/tests/interop/methods.py @@ -13,6 +13,14 @@ # limitations under the License. """Implementations of interoperability test methods.""" +# NOTE(lidiz) This module only exists in Bazel BUILD file, for more details +# please refer to comments in the "bazel_namespace_package_hack" module. +try: + from tests import bazel_namespace_package_hack + bazel_namespace_package_hack.sys_path_to_site_dir_hack() +except ImportError: + pass + import enum import json import os diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 560f6d3ddb3..29bb292c913 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -50,6 +50,16 @@ def _file_descriptor_to_proto(descriptor): class ReflectionServicerTest(unittest.TestCase): + # TODO(https://github.com/grpc/grpc/issues/17844) + # Bazel + Python 3 will result in creating two different instance of + # DESCRIPTOR for each message. So, the equal comparison between protobuf + # returned by stub and manually crafted protobuf will always fail. + def _assert_sequence_of_proto_equal(self, x, y): + self.assertSequenceEqual( + tuple(proto.SerializeToString() for proto in x), + tuple(proto.SerializeToString() for proto in y), + ) + def setUp(self): self._server = test_common.test_server() reflection.enable_server_reflection(_SERVICE_NAMES, self._server) @@ -84,7 +94,7 @@ class ReflectionServicerTest(unittest.TestCase): error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), )), ) - self.assertSequenceEqual(expected_responses, responses) + self._assert_sequence_of_proto_equal(expected_responses, responses) def testFileBySymbol(self): requests = ( @@ -108,7 +118,7 @@ class ReflectionServicerTest(unittest.TestCase): error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), )), ) - self.assertSequenceEqual(expected_responses, responses) + self._assert_sequence_of_proto_equal(expected_responses, responses) def testFileContainingExtension(self): requests = ( @@ -137,7 +147,7 @@ class ReflectionServicerTest(unittest.TestCase): error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), )), ) - self.assertSequenceEqual(expected_responses, responses) + self._assert_sequence_of_proto_equal(expected_responses, responses) def testExtensionNumbersOfType(self): requests = ( @@ -162,7 +172,7 @@ class ReflectionServicerTest(unittest.TestCase): error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), )), ) - self.assertSequenceEqual(expected_responses, responses) + self._assert_sequence_of_proto_equal(expected_responses, responses) def testListServices(self): requests = (reflection_pb2.ServerReflectionRequest(list_services='',),) @@ -173,7 +183,7 @@ class ReflectionServicerTest(unittest.TestCase): service=tuple( reflection_pb2.ServiceResponse(name=name) for name in _SERVICE_NAMES))),) - self.assertSequenceEqual(expected_responses, responses) + self._assert_sequence_of_proto_equal(expected_responses, responses) def testReflectionServiceName(self): self.assertEqual(reflection.SERVICE_NAME, diff --git a/src/python/grpcio_tests/tests/status/BUILD.bazel b/src/python/grpcio_tests/tests/status/BUILD.bazel index 937e50498e0..b163fe3975e 100644 --- a/src/python/grpcio_tests/tests/status/BUILD.bazel +++ b/src/python/grpcio_tests/tests/status/BUILD.bazel @@ -10,6 +10,7 @@ py_test( deps = [ "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_status/grpc_status:grpc_status", + "//src/python/grpcio_tests/tests:bazel_namespace_package_hack", "//src/python/grpcio_tests/tests/unit:test_common", "//src/python/grpcio_tests/tests/unit/framework/common:common", requirement('protobuf'), diff --git a/src/python/grpcio_tests/tests/status/_grpc_status_test.py b/src/python/grpcio_tests/tests/status/_grpc_status_test.py index 519c372a960..d1e9773b495 100644 --- a/src/python/grpcio_tests/tests/status/_grpc_status_test.py +++ b/src/python/grpcio_tests/tests/status/_grpc_status_test.py @@ -13,6 +13,14 @@ # limitations under the License. """Tests of grpc_status.""" +# NOTE(lidiz) This module only exists in Bazel BUILD file, for more details +# please refer to comments in the "bazel_namespace_package_hack" module. +try: + from tests import bazel_namespace_package_hack + bazel_namespace_package_hack.sys_path_to_site_dir_hack() +except ImportError: + pass + import unittest import logging diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl index 2ba1e07049c..e6fa5ed10e9 100644 --- a/third_party/py/python_configure.bzl +++ b/third_party/py/python_configure.bzl @@ -138,10 +138,13 @@ def _symlink_genrule_for_dir(repository_ctx, def _get_python_bin(repository_ctx): """Gets the python bin path.""" - python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) - if python_bin != None: - return python_bin - python_bin_path = repository_ctx.which("python") + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH, 'python') + if not repository_ctx.path(python_bin).exists: + # It's a command, use 'which' to find its path. + python_bin_path = repository_ctx.which(python_bin) + else: + # It's a path, use it as it is. + python_bin_path = python_bin if python_bin_path != None: return str(python_bin_path) _fail("Cannot find python in PATH, please make sure " + diff --git a/tools/bazel.rc b/tools/bazel.rc index 59e597b4723..99347495361 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -57,3 +57,7 @@ build:basicprof --copt=-DNDEBUG build:basicprof --copt=-O2 build:basicprof --copt=-DGRPC_BASIC_PROFILER build:basicprof --copt=-DGRPC_TIMERS_RDTSC + +build:python3 --python_path=python3 +build:python3 --force_python=PY3 +build:python3 --action_env=PYTHON_BIN_PATH=python3 diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index 156d65955ad..14989648a2a 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -25,3 +25,5 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc ${name}') cd /var/local/git/grpc/test bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/... +bazel clean --expunge +bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...