Merge pull request #17644 from lidizheng/bzl-py3

[gRPC] Enable Python 3 for Bazel to Run Tests

BUILD

@@ -63,6 +63,11 @@ config_setting(
     values = {"cpu": "x64_windows_msvc"},
 )
 
+config_setting(
+    name = "python3",
+    values = {"python_path": "python3"},
+)
+
 # This should be updated along with build.yaml
 g_stands_for = "gold"
 

src/python/grpcio/grpc/BUILD.bazel

@@ -15,9 +15,11 @@ py_library(
         "//src/python/grpcio/grpc/_cython:cygrpc",
         "//src/python/grpcio/grpc/experimental",
         "//src/python/grpcio/grpc/framework",
-        requirement('enum34'),
         requirement('six'),
-    ],
+    ] + select({
+        "//conditions:default": [requirement('enum34'),],
+        "//:python3": [],
+    }),
     data = [
         "//:grpc",
     ],

src/python/grpcio/grpc/framework/common/BUILD.bazel

@@ -13,15 +13,17 @@ py_library(
 py_library(
     name = "cardinality",
     srcs = ["cardinality.py"],
-    deps = [
+    deps = select({
-        requirement("enum34"),
+        "//conditions:default": [requirement('enum34'),],
-    ],
+        "//:python3": [],
+    }),
 )
 
 py_library(
     name = "style",
     srcs = ["style.py"],
-    deps = [
+    deps = select({
-        requirement("enum34"),
+        "//conditions:default": [requirement('enum34'),],
-    ],
+        "//:python3": [],
+    }),
 )

src/python/grpcio/grpc/framework/foundation/BUILD.bazel

@@ -23,9 +23,11 @@ py_library(
     name = "callable_util",
     srcs = ["callable_util.py"],
     deps = [
-        requirement("enum34"),
         requirement("six"),
-    ],
+    ] + select({
+        "//conditions:default": [requirement('enum34'),],
+        "//:python3": [],
+    }),
 )
 
 py_library(
@@ -39,9 +41,10 @@ py_library(
 py_library(
     name = "logging_pool",
     srcs = ["logging_pool.py"],
-    deps = [
+    deps = select({
-        requirement("futures"),
+        "//conditions:default": [requirement('futures'),],
-    ],
+        "//:python3": [],
+    }),
 )
 
 py_library(

src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel

@@ -15,15 +15,18 @@ py_library(
     srcs = ["base.py"],
     deps = [
         "//src/python/grpcio/grpc/framework/foundation:abandonment",
-        requirement("enum34"),
         requirement("six"),
-    ],
+    ] + select({
+        "//conditions:default": [requirement('enum34'),],
+        "//:python3": [],
+    }),
 )
 
 py_library(
     name = "utilities",
     srcs = ["utilities.py"],
-    deps = [
+    deps = select({
-        requirement("enum34"),
+        "//conditions:default": [requirement('enum34'),],
-    ],
+        "//:python3": [],
+    }),
 )

src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel

@@ -16,9 +16,11 @@ py_library(
     deps = [
         "//src/python/grpcio/grpc/framework/foundation",
         "//src/python/grpcio/grpc/framework/common",
-        requirement("enum34"),
         requirement("six"),
-    ],
+    ] + select({
+        "//conditions:default": [requirement('enum34'),],
+        "//:python3": [],
+    }),
 )
 
 py_library(

src/python/grpcio_status/grpc_status/rpc_status.py

@@ -17,11 +17,6 @@ import collections
 
 import grpc
 
-# TODO(https://github.com/bazelbuild/bazel/issues/6844)
-# Due to Bazel issue, the namespace packages won't resolve correctly.
-# Adding this unused-import as a workaround to avoid module-not-found error
-# under Bazel builds.
-import google.protobuf  # pylint: disable=unused-import
 from google.rpc import status_pb2
 
 _CODE_TO_GRPC_CODE_MAPPING = {x.value[0]: x for x in grpc.StatusCode}

src/python/grpcio_tests/tests/BUILD.bazel

@@ -0,0 +1,8 @@
+py_library(
+    name = "bazel_namespace_package_hack",
+    srcs = ["bazel_namespace_package_hack.py"],
+    visibility = [
+        "//src/python/grpcio_tests/tests/status:__subpackages__",
+        "//src/python/grpcio_tests/tests/interop:__subpackages__",
+    ],
+)

src/python/grpcio_tests/tests/bazel_namespace_package_hack.py

@@ -0,0 +1,32 @@
+# Copyright 2019 The gRPC Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import site
+import sys
+
+
+# TODO(https://github.com/bazelbuild/bazel/issues/6844) Bazel failed to
+# interpret namespace packages correctly. This monkey patch will force the
+# Python process to parse the .pth file in the sys.path to resolve namespace
+# package in the right place.
+# Analysis in depth: https://github.com/bazelbuild/rules_python/issues/55
+def sys_path_to_site_dir_hack():
+    """Add valid sys.path item to site directory to parse the .pth files."""
+    for item in sys.path:
+        if os.path.exists(item):
+            # The only difference between sys.path and site-directory is
+            # whether the .pth file will be parsed or not. A site-directory
+            # will always exist in sys.path, but not another way around.
+            site.addsitedir(item)

src/python/grpcio_tests/tests/interop/BUILD.bazel

@@ -29,17 +29,20 @@ py_library(
     srcs = ["methods.py"],
     deps = [
         "//src/python/grpcio/grpc:grpcio",
+        "//src/python/grpcio_tests/tests:bazel_namespace_package_hack",
         "//src/proto/grpc/testing:py_empty_proto",
         "//src/proto/grpc/testing:py_messages_proto",
         "//src/proto/grpc/testing:py_test_proto",
         requirement('google-auth'),
         requirement('requests'),
-        requirement('enum34'),
         requirement('urllib3'),
         requirement('chardet'),
         requirement('certifi'),
         requirement('idna'),
-    ],
+    ] + select({
+        "//conditions:default": [requirement('enum34'),],
+        "//:python3": [],
+    }),
     imports=["../../",],
 )
 

src/python/grpcio_tests/tests/interop/methods.py

@@ -13,6 +13,9 @@
 # limitations under the License.
 """Implementations of interoperability test methods."""
 
+from tests import bazel_namespace_package_hack
+bazel_namespace_package_hack.sys_path_to_site_dir_hack()
+
 import enum
 import json
 import os

src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py

@@ -50,6 +50,16 @@ def _file_descriptor_to_proto(descriptor):
 
 class ReflectionServicerTest(unittest.TestCase):
 
+    # TODO(https://github.com/grpc/grpc/issues/17844)
+    # Bazel + Python 3 will result in creating two different instance of
+    # DESCRIPTOR for each message. So, the equal comparision between protobuf
+    # returned by stub and manually crafted protobuf will always fail.
+    def _assert_sequence_of_proto_equal(self, x, y):
+        self.assertSequenceEqual(
+            list(map(lambda x: x.SerializeToString(), x)),
+            list(map(lambda x: x.SerializeToString(), y)),
+        )
+
     def setUp(self):
         self._server = test_common.test_server()
         reflection.enable_server_reflection(_SERVICE_NAMES, self._server)
@@ -84,7 +94,7 @@ class ReflectionServicerTest(unittest.TestCase):
                     error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(),
                 )),
         )
-        self.assertSequenceEqual(expected_responses, responses)
+        self._assert_sequence_of_proto_equal(expected_responses, responses)
 
     def testFileBySymbol(self):
         requests = (
@@ -108,7 +118,7 @@ class ReflectionServicerTest(unittest.TestCase):
                     error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(),
                 )),
         )
-        self.assertSequenceEqual(expected_responses, responses)
+        self._assert_sequence_of_proto_equal(expected_responses, responses)
 
     def testFileContainingExtension(self):
         requests = (
@@ -137,7 +147,7 @@ class ReflectionServicerTest(unittest.TestCase):
                     error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(),
                 )),
         )
-        self.assertSequenceEqual(expected_responses, responses)
+        self._assert_sequence_of_proto_equal(expected_responses, responses)
 
     def testExtensionNumbersOfType(self):
         requests = (
@@ -162,7 +172,7 @@ class ReflectionServicerTest(unittest.TestCase):
                     error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(),
                 )),
         )
-        self.assertSequenceEqual(expected_responses, responses)
+        self._assert_sequence_of_proto_equal(expected_responses, responses)
 
     def testListServices(self):
         requests = (reflection_pb2.ServerReflectionRequest(list_services='',),)
@@ -173,7 +183,7 @@ class ReflectionServicerTest(unittest.TestCase):
                 service=tuple(
                     reflection_pb2.ServiceResponse(name=name)
                     for name in _SERVICE_NAMES))),)
-        self.assertSequenceEqual(expected_responses, responses)
+        self._assert_sequence_of_proto_equal(expected_responses, responses)
 
     def testReflectionServiceName(self):
         self.assertEqual(reflection.SERVICE_NAME,

src/python/grpcio_tests/tests/status/BUILD.bazel

@@ -10,6 +10,7 @@ py_test(
     deps = [
         "//src/python/grpcio/grpc:grpcio",
         "//src/python/grpcio_status/grpc_status:grpc_status",
+        "//src/python/grpcio_tests/tests:bazel_namespace_package_hack",
         "//src/python/grpcio_tests/tests/unit:test_common",
         "//src/python/grpcio_tests/tests/unit/framework/common:common",
         requirement('protobuf'),

src/python/grpcio_tests/tests/status/_grpc_status_test.py

@@ -13,6 +13,9 @@
 # limitations under the License.
 """Tests of grpc_status."""
 
+from tests import bazel_namespace_package_hack
+bazel_namespace_package_hack.sys_path_to_site_dir_hack()
+
 import unittest
 
 import logging

third_party/py/python_configure.bzl

@@ -138,10 +138,13 @@ def _symlink_genrule_for_dir(repository_ctx,
 
 def _get_python_bin(repository_ctx):
     """Gets the python bin path."""
-    python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH)
+    python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH, 'python')
-    if python_bin != None:
+    if not '/' in python_bin and not '\\' in python_bin:
-        return python_bin
+        # It's a command, use 'which' to find its path.
-    python_bin_path = repository_ctx.which("python")
+        python_bin_path = repository_ctx.which(python_bin)
+    else:
+        # It's a path, use it as it is.
+        python_bin_path = python_bin
     if python_bin_path != None:
         return str(python_bin_path)
     _fail("Cannot find python in PATH, please make sure " +

tools/bazel.rc

@@ -57,3 +57,7 @@ build:basicprof --copt=-DNDEBUG
 build:basicprof --copt=-O2
 build:basicprof --copt=-DGRPC_BASIC_PROFILER
 build:basicprof --copt=-DGRPC_TIMERS_RDTSC
+
+build:python3 --python_path=python3
+build:python3 --force_python=PY3
+build:python3 --action_env=PYTHON_BIN_PATH=python3

tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh

@@ -25,3 +25,5 @@ git clone /var/local/jenkins/grpc /var/local/git/grpc
 ${name}')
 cd /var/local/git/grpc/test
 bazel test --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...
+bazel clean --expunge
+bazel test --config=python3 --spawn_strategy=standalone --genrule_strategy=standalone --test_output=errors //src/python/...