Create Bazel gevent test harness (#27507)
* WIP * Add gevent test suite run under Bazel. * Fix things up * Yapf * Fix up Bazel files * Make py_grpc_test fancier * Attempt to fix Windows RBE * Attempt to kick GitHub * Fix Python 2 runs * Yet more fixes * And the patch file too * I am an idiot * Mark gevent tests flaky * Try to make rules_python more tolerant * Typo * Exclude reconnect test from gevent * Remove unnecessary parts of patch * Buildifier * You saw nothing * isort * Move py_grpc_test to an internal-only file * Review comments * More reviewer comments * Reviewpull/27644/head
Richard Belleville
4 years ago
committed by
GitHub
parent
76dd0474e4
commit
7aa43b7a55
20 changed files with 297 additions and 14 deletions
@ -0,0 +1,60 @@ |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
import grpc |
||||
import unittest |
||||
import sys |
||||
import os |
||||
import pkgutil |
||||
|
||||
from typing import Sequence |
||||
|
||||
class SingleLoader(object): |
||||
def __init__(self, pattern: str): |
||||
loader = unittest.TestLoader() |
||||
self.suite = unittest.TestSuite() |
||||
tests = [] |
||||
for importer, module_name, is_package in pkgutil.walk_packages([os.path.dirname(os.path.relpath(__file__))]): |
||||
if pattern in module_name: |
||||
module = importer.find_module(module_name).load_module(module_name) |
||||
tests.append(loader.loadTestsFromModule(module)) |
||||
if len(tests) != 1: |
||||
raise AssertionError("Expected only 1 test module. Found {}".format(tests)) |
||||
self.suite.addTest(tests[0]) |
||||
|
||||
|
||||
def loadTestsFromNames(self, names: Sequence[str], module: str = None) -> unittest.TestSuite: |
||||
return self.suite |
||||
|
||||
if __name__ == "__main__": |
||||
from gevent import monkey |
||||
|
||||
monkey.patch_all() |
||||
|
||||
import grpc.experimental.gevent |
||||
grpc.experimental.gevent.init_gevent() |
||||
import gevent |
||||
|
||||
if len(sys.argv) != 2: |
||||
print(f"USAGE: {sys.argv[0]} TARGET_MODULE", file=sys.stderr) |
||||
|
||||
target_module = sys.argv[1] |
||||
|
||||
loader = SingleLoader(target_module) |
||||
runner = unittest.TextTestRunner() |
||||
|
||||
result = gevent.spawn(runner.run, loader.suite) |
||||
result.join() |
||||
if not result.value.wasSuccessful(): |
||||
sys.exit("Test failure.") |
||||
@ -0,0 +1,71 @@ |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
load("@grpc_python_dependencies//:requirements.bzl", "requirement") |
||||
|
||||
_GRPC_LIB = "//src/python/grpcio/grpc:grpcio" |
||||
|
||||
_COPIED_MAIN_SUFFIX = ".gevent.main" |
||||
|
||||
def py_grpc_gevent_test( |
||||
name, |
||||
srcs, |
||||
main = None, |
||||
deps = None, |
||||
data = None, |
||||
**kwargs): |
||||
if main == None: |
||||
if len(srcs) != 1: |
||||
fail("When main is not provided, srcs must be of size 1.") |
||||
main = srcs[0] |
||||
deps = [] if deps == None else deps |
||||
data = [] if data == None else data |
||||
lib_name = name + ".gevent.lib" |
||||
supplied_python_version = kwargs.pop("python_version", "") |
||||
if supplied_python_version and supplied_python_version != "PY3": |
||||
fail("py_grpc_gevent_test only supports python_version=PY3") |
||||
native.py_library( |
||||
name = lib_name, |
||||
srcs = srcs, |
||||
) |
||||
augmented_deps = deps + [ |
||||
":{}".format(lib_name), |
||||
requirement("gevent"), |
||||
] |
||||
if _GRPC_LIB not in augmented_deps: |
||||
augmented_deps.append(_GRPC_LIB) |
||||
|
||||
# The main file needs to be in the same package as the test file. |
||||
copied_main_name = name + _COPIED_MAIN_SUFFIX |
||||
copied_main_filename = copied_main_name + ".py" |
||||
native.genrule( |
||||
name = copied_main_name, |
||||
srcs = ["//bazel:_gevent_test_main.py"], |
||||
outs = [copied_main_filename], |
||||
cmd = "cp $< $@", |
||||
) |
||||
|
||||
# TODO(https://github.com/grpc/grpc/issues/27542): Remove once gevent is deemed non-flaky. |
||||
if "flaky" in kwargs: |
||||
kwargs.pop("flaky") |
||||
|
||||
native.py_test( |
||||
name = name + ".gevent", |
||||
args = [name], |
||||
deps = augmented_deps, |
||||
srcs = [copied_main_filename], |
||||
main = copied_main_filename, |
||||
python_version = "PY3", |
||||
flaky = True, |
||||
**kwargs |
||||
) |
||||
@ -0,0 +1,35 @@ |
||||
# Copyright 2021 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
"""Python-related rules intended only for use internal to the repo.""" |
||||
|
||||
load("//bazel:gevent_test.bzl", "py_grpc_gevent_test") |
||||
load("//bazel:python_rules.bzl", "py2and3_test") |
||||
|
||||
def internal_py_grpc_test(name, **kwargs): |
||||
"""Runs a test under all supported environments.""" |
||||
py2and3_test(name, **kwargs) |
||||
py_grpc_gevent_test(name, **kwargs) |
||||
|
||||
suite_kwargs = {} |
||||
if "visibility" in kwargs: |
||||
suite_kwargs["visibility"] = kwargs["visibility"] |
||||
|
||||
native.test_suite( |
||||
name = name, |
||||
tests = [ |
||||
name + ".both_pythons", |
||||
name + ".gevent", |
||||
], |
||||
**suite_kwargs |
||||
) |
||||
@ -0,0 +1,76 @@ |
||||
diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl
|
||||
index c3007e1..f8a9234 100644
|
||||
--- a/python/pip_install/pip_repository.bzl
|
||||
+++ b/python/pip_install/pip_repository.bzl
|
||||
@@ -39,7 +39,8 @@ def _resolve_python_interpreter(rctx):
|
||||
if "/" not in python_interpreter:
|
||||
python_interpreter = rctx.which(python_interpreter)
|
||||
if not python_interpreter:
|
||||
- fail("python interpreter not found")
|
||||
+ print("WARNING: python interpreter not found. Python targets will not be functional")
|
||||
+ return ""
|
||||
return python_interpreter
|
||||
|
||||
def _parse_optional_attrs(rctx, args):
|
||||
@@ -93,13 +94,49 @@ def _parse_optional_attrs(rctx, args):
|
||||
|
||||
return args
|
||||
|
||||
+def _generate_stub_requirements_bzl(rctx):
|
||||
+ contents = """\
|
||||
+def requirement(name):
|
||||
+ return "@{repo}//:empty"
|
||||
+""".format(repo=rctx.attr.name)
|
||||
+ rctx.file("requirements.bzl", contents)
|
||||
+
|
||||
_BUILD_FILE_CONTENTS = """\
|
||||
package(default_visibility = ["//visibility:public"])
|
||||
|
||||
# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
|
||||
exports_files(["requirements.bzl"])
|
||||
+
|
||||
+py_library(
|
||||
+ name = "empty",
|
||||
+ srcs = [],
|
||||
+)
|
||||
"""
|
||||
|
||||
+def _python_version_info(rctx, python_interpreter, info_index):
|
||||
+ cmd = [
|
||||
+ python_interpreter,
|
||||
+ "-c",
|
||||
+ "from __future__ import print_function; import sys; print(sys.version_info[{}])".format(info_index)
|
||||
+ ]
|
||||
+ result = rctx.execute(cmd)
|
||||
+ if result.stderr or not result.stdout:
|
||||
+ print("WARNING: Failed to get version info from {}".format(python_interpreter))
|
||||
+ return None
|
||||
+ return int(result.stdout.strip())
|
||||
+
|
||||
+def _python_version_supported(rctx, python_interpreter):
|
||||
+ major_version = _python_version_info(rctx, python_interpreter, 0)
|
||||
+ minor_version = _python_version_info(rctx, python_interpreter, 1)
|
||||
+ if major_version == None or minor_version == None:
|
||||
+ print("WARNING: Failed to get Python version of {}".format(python_interpreter))
|
||||
+ return False
|
||||
+ if (major_version != 3 or minor_version < 6):
|
||||
+ print("WARNING: {} is of version {}.{}. This version is unsupported.".format(python_interpreter, major_version, minor_version))
|
||||
+ return False
|
||||
+ return True
|
||||
+
|
||||
+
|
||||
def _pip_repository_impl(rctx):
|
||||
python_interpreter = _resolve_python_interpreter(rctx)
|
||||
|
||||
@@ -109,6 +146,11 @@ def _pip_repository_impl(rctx):
|
||||
# We need a BUILD file to load the generated requirements.bzl
|
||||
rctx.file("BUILD.bazel", _BUILD_FILE_CONTENTS)
|
||||
|
||||
+ # Check if python interpreter has minimum required version.
|
||||
+ if not python_interpreter or not _python_version_supported(rctx, python_interpreter):
|
||||
+ _generate_stub_requirements_bzl(rctx)
|
||||
+ return
|
||||
+
|
||||
pypath = _construct_pypath(rctx)
|
||||
|
||||
if rctx.attr.incremental:
|
||||
Loading…
Reference in new issue