diff --git a/WORKSPACE b/WORKSPACE index 52847be6068..da870cbebcd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -95,15 +95,9 @@ rbe_autoconfig( ), ) -load("@io_bazel_rules_python//python:pip.bzl", "pip_import", "pip_repositories") +load("@io_bazel_rules_python//python:pip.bzl", "pip_install") -pip_import( +pip_install( name = "grpc_python_dependencies", requirements = "@com_github_grpc_grpc//:requirements.bazel.txt", ) - -load("@grpc_python_dependencies//:requirements.bzl", "pip_install") - -pip_repositories() - -pip_install() diff --git a/bazel/BUILD b/bazel/BUILD index f94b9824360..aa6cb341dfe 100644 --- a/bazel/BUILD +++ b/bazel/BUILD @@ -17,3 +17,8 @@ licenses(["notice"]) package(default_visibility = ["//:__subpackages__"]) load(":cc_grpc_library.bzl", "cc_grpc_library") + +filegroup( + name = "_gevent_test_main", + srcs = ["_gevent_test_main.py"], +) diff --git a/bazel/_gevent_test_main.py b/bazel/_gevent_test_main.py new file mode 100644 index 00000000000..52629c61d3c --- /dev/null +++ b/bazel/_gevent_test_main.py @@ -0,0 +1,60 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import grpc +import unittest +import sys +import os +import pkgutil + +from typing import Sequence + +class SingleLoader(object): + def __init__(self, pattern: str): + loader = unittest.TestLoader() + self.suite = unittest.TestSuite() + tests = [] + for importer, module_name, is_package in pkgutil.walk_packages([os.path.dirname(os.path.relpath(__file__))]): + if pattern in module_name: + module = importer.find_module(module_name).load_module(module_name) + tests.append(loader.loadTestsFromModule(module)) + if len(tests) != 1: + raise AssertionError("Expected only 1 test module. Found {}".format(tests)) + self.suite.addTest(tests[0]) + + + def loadTestsFromNames(self, names: Sequence[str], module: str = None) -> unittest.TestSuite: + return self.suite + +if __name__ == "__main__": + from gevent import monkey + + monkey.patch_all() + + import grpc.experimental.gevent + grpc.experimental.gevent.init_gevent() + import gevent + + if len(sys.argv) != 2: + print(f"USAGE: {sys.argv[0]} TARGET_MODULE", file=sys.stderr) + + target_module = sys.argv[1] + + loader = SingleLoader(target_module) + runner = unittest.TextTestRunner() + + result = gevent.spawn(runner.run, loader.suite) + result.join() + if not result.value.wasSuccessful(): + sys.exit("Test failure.") diff --git a/bazel/gevent_test.bzl b/bazel/gevent_test.bzl new file mode 100644 index 00000000000..55444d32195 --- /dev/null +++ b/bazel/gevent_test.bzl @@ -0,0 +1,71 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +_GRPC_LIB = "//src/python/grpcio/grpc:grpcio" + +_COPIED_MAIN_SUFFIX = ".gevent.main" + +def py_grpc_gevent_test( + name, + srcs, + main = None, + deps = None, + data = None, + **kwargs): + if main == None: + if len(srcs) != 1: + fail("When main is not provided, srcs must be of size 1.") + main = srcs[0] + deps = [] if deps == None else deps + data = [] if data == None else data + lib_name = name + ".gevent.lib" + supplied_python_version = kwargs.pop("python_version", "") + if supplied_python_version and supplied_python_version != "PY3": + fail("py_grpc_gevent_test only supports python_version=PY3") + native.py_library( + name = lib_name, + srcs = srcs, + ) + augmented_deps = deps + [ + ":{}".format(lib_name), + requirement("gevent"), + ] + if _GRPC_LIB not in augmented_deps: + augmented_deps.append(_GRPC_LIB) + + # The main file needs to be in the same package as the test file. + copied_main_name = name + _COPIED_MAIN_SUFFIX + copied_main_filename = copied_main_name + ".py" + native.genrule( + name = copied_main_name, + srcs = ["//bazel:_gevent_test_main.py"], + outs = [copied_main_filename], + cmd = "cp $< $@", + ) + + # TODO(https://github.com/grpc/grpc/issues/27542): Remove once gevent is deemed non-flaky. + if "flaky" in kwargs: + kwargs.pop("flaky") + + native.py_test( + name = name + ".gevent", + args = [name], + deps = augmented_deps, + srcs = [copied_main_filename], + main = copied_main_filename, + python_version = "PY3", + flaky = True, + **kwargs + ) diff --git a/bazel/grpc_python_deps.bzl b/bazel/grpc_python_deps.bzl index 4c47c5c1186..bbbaa2a1bb9 100644 --- a/bazel/grpc_python_deps.bzl +++ b/bazel/grpc_python_deps.bzl @@ -49,8 +49,10 @@ def grpc_python_deps(): if "io_bazel_rules_python" not in native.existing_rules(): http_archive( name = "io_bazel_rules_python", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.1/rules_python-0.0.1.tar.gz", - sha256 = "aa96a691d3a8177f3215b14b0edc9641787abaaa30363a080165d06ab65e1161", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.4.0/rules_python-0.4.0.tar.gz", + sha256 = "954aa89b491be4a083304a2cb838019c8b8c3720a7abb9c4cb81ac7a24230cea", + patches = ["//third_party:rules_python.patch"], + patch_args = ["-p1"], ) python_configure(name = "local_config_python") diff --git a/bazel/internal_python_rules.bzl b/bazel/internal_python_rules.bzl new file mode 100644 index 00000000000..6f3c8751b0d --- /dev/null +++ b/bazel/internal_python_rules.bzl @@ -0,0 +1,35 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Python-related rules intended only for use internal to the repo.""" + +load("//bazel:gevent_test.bzl", "py_grpc_gevent_test") +load("//bazel:python_rules.bzl", "py2and3_test") + +def internal_py_grpc_test(name, **kwargs): + """Runs a test under all supported environments.""" + py2and3_test(name, **kwargs) + py_grpc_gevent_test(name, **kwargs) + + suite_kwargs = {} + if "visibility" in kwargs: + suite_kwargs["visibility"] = kwargs["visibility"] + + native.test_suite( + name = name, + tests = [ + name + ".both_pythons", + name + ".gevent", + ], + **suite_kwargs + ) diff --git a/bazel/python_rules.bzl b/bazel/python_rules.bzl index df067d36abb..68137bd21d8 100644 --- a/bazel/python_rules.bzl +++ b/bazel/python_rules.bzl @@ -268,6 +268,7 @@ def py_grpc_library( **kwargs ) +# TODO(https://github.com/grpc/grpc/issues/27543): Remove once Python 2 is no longer supported. def py2and3_test( name, py_test = native.py_test, @@ -297,7 +298,7 @@ def py2and3_test( suite_kwargs["visibility"] = kwargs["visibility"] native.test_suite( - name = name, + name = name + ".both_pythons", tests = names, **suite_kwargs ) diff --git a/requirements.bazel.txt b/requirements.bazel.txt index 57deb95420a..e24eca17797 100644 --- a/requirements.bazel.txt +++ b/requirements.bazel.txt @@ -14,3 +14,6 @@ chardet==3.0.4 certifi==2017.4.17 idna==2.7 googleapis-common-protos==1.5.5 +gevent==21.1.2 +zope.event==4.5.0 +setuptools==44.1.1 diff --git a/src/python/grpcio_tests/tests/unit/BUILD.bazel b/src/python/grpcio_tests/tests/unit/BUILD.bazel index 67d7a900318..bf8bdef0e38 100644 --- a/src/python/grpcio_tests/tests/unit/BUILD.bazel +++ b/src/python/grpcio_tests/tests/unit/BUILD.bazel @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -load("//bazel:python_rules.bzl", "py2and3_test") +load("//bazel:internal_python_rules.bzl", "internal_py_grpc_test") package(default_visibility = ["//visibility:public"]) @@ -105,7 +105,7 @@ py_library( ) [ - py2and3_test( + internal_py_grpc_test( name = test_file_name[:-3], size = "small", srcs = [test_file_name], @@ -133,7 +133,7 @@ py_library( for test_file_name in GRPCIO_TESTS_UNIT ] -py2and3_test( +internal_py_grpc_test( name = "_dynamic_stubs_test", size = "small", srcs = ["_dynamic_stubs_test.py"], @@ -144,6 +144,7 @@ py2and3_test( imports = ["../../"], main = "_dynamic_stubs_test.py", deps = [ + ":test_common", "//src/python/grpcio/grpc:grpcio", "//src/python/grpcio_tests/tests/testing", "//tools/distrib/python/grpcio_tools:grpc_tools", diff --git a/src/python/grpcio_tests/tests/unit/_compression_test.py b/src/python/grpcio_tests/tests/unit/_compression_test.py index 9789afbfe7f..cd55db58baf 100644 --- a/src/python/grpcio_tests/tests/unit/_compression_test.py +++ b/src/python/grpcio_tests/tests/unit/_compression_test.py @@ -259,6 +259,8 @@ def _stream_stream_client(channel, multicallable_kwargs, message): i, response)) +@unittest.skipIf(test_common.running_under_gevent(), + "This test is nondeterministic under gevent.") class CompressionTest(unittest.TestCase): def assertCompressed(self, compression_ratio): diff --git a/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py b/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py index 9c94be5a1da..128ec514d06 100644 --- a/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py +++ b/src/python/grpcio_tests/tests/unit/_contextvars_propagation_test.py @@ -97,6 +97,8 @@ else: # TODO(https://github.com/grpc/grpc/issues/22257) @unittest.skipIf(os.name == "nt", "LocalCredentials not supported on Windows.") +@unittest.skipIf(test_common.running_under_gevent(), + "ThreadLocals do not work under gevent.") class ContextVarsPropagationTest(unittest.TestCase): def test_propagation_to_auth_plugin(self): diff --git a/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py b/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py index 4645f0b633f..933a8143eda 100644 --- a/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py +++ b/src/python/grpcio_tests/tests/unit/_dynamic_stubs_test.py @@ -21,6 +21,8 @@ import os import sys import unittest +from tests.unit import test_common + _DATA_DIR = os.path.join("tests", "unit", "data") @@ -124,6 +126,8 @@ def _test_grpc_tools_unimportable(): # when they do not come from the "__main__" module, so this test passes # if run directly on Windows, but not if started by the test runner. @unittest.skipIf(os.name == "nt", "Windows multiprocessing unsupported") +@unittest.skipIf(test_common.running_under_gevent(), + "Import paths do not work with gevent runner.") class DynamicStubTest(unittest.TestCase): def test_sunny_day(self): diff --git a/src/python/grpcio_tests/tests/unit/_local_credentials_test.py b/src/python/grpcio_tests/tests/unit/_local_credentials_test.py index ce92feed4b3..5351a2b4cc1 100644 --- a/src/python/grpcio_tests/tests/unit/_local_credentials_test.py +++ b/src/python/grpcio_tests/tests/unit/_local_credentials_test.py @@ -19,6 +19,8 @@ import unittest import grpc +from tests.unit import test_common + class _GenericHandler(grpc.GenericRpcHandler): @@ -56,6 +58,8 @@ class LocalCredentialsTest(unittest.TestCase): @unittest.skipIf(os.name == 'nt', 'Unix Domain Socket is not supported on Windows') + @unittest.skipIf(test_common.running_under_gevent(), + 'UDS not supported under gevent.') def test_uds(self): server_addr = 'unix:/tmp/grpc_fullstack_test' channel_creds = grpc.local_channel_credentials( diff --git a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py index 89c028b307b..87441833c17 100644 --- a/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py +++ b/src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py @@ -188,6 +188,8 @@ def _generic_handler(servicer): return grpc.method_handlers_generic_handler(_SERVICE, method_handlers) +@unittest.skipIf(test_common.running_under_gevent(), + "Causes deadlock in gevent.") class MetadataCodeDetailsTest(unittest.TestCase): def setUp(self): diff --git a/src/python/grpcio_tests/tests/unit/_reconnect_test.py b/src/python/grpcio_tests/tests/unit/_reconnect_test.py index 90d010b9360..62ab5a58fc6 100644 --- a/src/python/grpcio_tests/tests/unit/_reconnect_test.py +++ b/src/python/grpcio_tests/tests/unit/_reconnect_test.py @@ -21,6 +21,7 @@ import unittest import grpc from grpc.framework.foundation import logging_pool +from tests.unit import test_common from tests.unit.framework.common import bound_socket from tests.unit.framework.common import test_constants @@ -34,6 +35,8 @@ def _handle_unary_unary(unused_request, unused_servicer_context): return _RESPONSE +@unittest.skipIf(test_common.running_under_gevent(), + "Test is nondeterministic under gevent.") class ReconnectTest(unittest.TestCase): def test_reconnect(self): diff --git a/src/python/grpcio_tests/tests/unit/_rpc_part_1_test.py b/src/python/grpcio_tests/tests/unit/_rpc_part_1_test.py index 1f85cd6f91e..0ffa9eff94f 100644 --- a/src/python/grpcio_tests/tests/unit/_rpc_part_1_test.py +++ b/src/python/grpcio_tests/tests/unit/_rpc_part_1_test.py @@ -22,6 +22,7 @@ import unittest import grpc from grpc.framework.foundation import logging_pool +from tests.unit import test_common from tests.unit._rpc_test_helpers import BaseRPCTest from tests.unit._rpc_test_helpers import Callback from tests.unit._rpc_test_helpers import TIMEOUT_SHORT @@ -36,6 +37,8 @@ from tests.unit._rpc_test_helpers import unary_unary_multi_callable from tests.unit.framework.common import test_constants +@unittest.skipIf(test_common.running_under_gevent(), + "This test is nondeterministic under gevent.") class RPCPart1Test(BaseRPCTest, unittest.TestCase): def testExpiredStreamRequestBlockingUnaryResponse(self): diff --git a/src/python/grpcio_tests/tests/unit/_rpc_part_2_test.py b/src/python/grpcio_tests/tests/unit/_rpc_part_2_test.py index a8e6ddeb534..6a82a9588f6 100644 --- a/src/python/grpcio_tests/tests/unit/_rpc_part_2_test.py +++ b/src/python/grpcio_tests/tests/unit/_rpc_part_2_test.py @@ -22,6 +22,7 @@ import unittest import grpc from grpc.framework.foundation import logging_pool +from tests.unit import test_common from tests.unit._rpc_test_helpers import BaseRPCTest from tests.unit._rpc_test_helpers import Callback from tests.unit._rpc_test_helpers import TIMEOUT_SHORT @@ -36,6 +37,8 @@ from tests.unit._rpc_test_helpers import unary_unary_multi_callable from tests.unit.framework.common import test_constants +@unittest.skipIf(test_common.running_under_gevent(), + "Causes deadlock under gevent.") class RPCPart2Test(BaseRPCTest, unittest.TestCase): def testDefaultThreadPoolIsUsed(self): diff --git a/src/python/grpcio_tests/tests/unit/test_common.py b/src/python/grpcio_tests/tests/unit/test_common.py index 3b1e3442478..dae69cbcebc 100644 --- a/src/python/grpcio_tests/tests/unit/test_common.py +++ b/src/python/grpcio_tests/tests/unit/test_common.py @@ -132,3 +132,14 @@ class WaitGroup(object): while self.count > 0: self.cv.wait() self.cv.release() + + +def running_under_gevent(): + try: + from gevent import monkey + import gevent.socket + except ImportError: + return False + else: + import socket + return socket.socket is gevent.socket.socket diff --git a/third_party/BUILD b/third_party/BUILD index 272d2396003..0ac04511a57 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -12,4 +12,5 @@ exports_files([ "futures.BUILD", "libuv.BUILD", "protobuf.patch", + "rules_python.patch", ]) diff --git a/third_party/rules_python.patch b/third_party/rules_python.patch new file mode 100644 index 00000000000..3b2b5b8d0df --- /dev/null +++ b/third_party/rules_python.patch @@ -0,0 +1,76 @@ +diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl +index c3007e1..f8a9234 100644 +--- a/python/pip_install/pip_repository.bzl ++++ b/python/pip_install/pip_repository.bzl +@@ -39,7 +39,8 @@ def _resolve_python_interpreter(rctx): + if "/" not in python_interpreter: + python_interpreter = rctx.which(python_interpreter) + if not python_interpreter: +- fail("python interpreter not found") ++ print("WARNING: python interpreter not found. Python targets will not be functional") ++ return "" + return python_interpreter + + def _parse_optional_attrs(rctx, args): +@@ -93,13 +94,49 @@ def _parse_optional_attrs(rctx, args): + + return args + ++def _generate_stub_requirements_bzl(rctx): ++ contents = """\ ++def requirement(name): ++ return "@{repo}//:empty" ++""".format(repo=rctx.attr.name) ++ rctx.file("requirements.bzl", contents) ++ + _BUILD_FILE_CONTENTS = """\ + package(default_visibility = ["//visibility:public"]) + + # Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it + exports_files(["requirements.bzl"]) ++ ++py_library( ++ name = "empty", ++ srcs = [], ++) + """ + ++def _python_version_info(rctx, python_interpreter, info_index): ++ cmd = [ ++ python_interpreter, ++ "-c", ++ "from __future__ import print_function; import sys; print(sys.version_info[{}])".format(info_index) ++ ] ++ result = rctx.execute(cmd) ++ if result.stderr or not result.stdout: ++ print("WARNING: Failed to get version info from {}".format(python_interpreter)) ++ return None ++ return int(result.stdout.strip()) ++ ++def _python_version_supported(rctx, python_interpreter): ++ major_version = _python_version_info(rctx, python_interpreter, 0) ++ minor_version = _python_version_info(rctx, python_interpreter, 1) ++ if major_version == None or minor_version == None: ++ print("WARNING: Failed to get Python version of {}".format(python_interpreter)) ++ return False ++ if (major_version != 3 or minor_version < 6): ++ print("WARNING: {} is of version {}.{}. This version is unsupported.".format(python_interpreter, major_version, minor_version)) ++ return False ++ return True ++ ++ + def _pip_repository_impl(rctx): + python_interpreter = _resolve_python_interpreter(rctx) + +@@ -109,6 +146,11 @@ def _pip_repository_impl(rctx): + # We need a BUILD file to load the generated requirements.bzl + rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS) + ++ # Check if python interpreter has minimum required version. ++ if not python_interpreter or not _python_version_supported(rctx, python_interpreter): ++ _generate_stub_requirements_bzl(rctx) ++ return ++ + pypath = _construct_pypath(rctx) + + if rctx.attr.incremental: