[bazel] Add more bazelified tests (#34005)

- some cleanup
- add bazel distribtests (to run the existing
test_single_bazel_version.sh locally under a docker container created by
RBE).
- add c++ distribtests
- rename `grpc_run_tests_py_test` macro to `grpc_run_tests_harness_test`
- as requested previously to avoid confusion with `*_py_test` rules
run_test_verbose_success
Jan Tattermusch 2 years ago committed by GitHub
parent f3419f8373
commit 5eb227936e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 5
      tools/bazelify_tests/BUILD
  2. 140
      tools/bazelify_tests/build_defs.bzl
  3. 36
      tools/bazelify_tests/grpc_run_bazel_distribtest_test.sh
  4. 34
      tools/bazelify_tests/grpc_run_cpp_distribtest_test.sh
  5. 0
      tools/bazelify_tests/grpc_run_tests_harness_test.sh
  6. 114
      tools/bazelify_tests/test/BUILD
  7. 10
      tools/bazelify_tests/test/portability_tests.bzl

@ -23,8 +23,9 @@ grpc_package(
)
exports_files([
"grpc_run_tests_py_test.sh",
"grpc_build_artifact_task.sh",
"grpc_run_tests_harness_test.sh",
"grpc_run_bazel_distribtest_test.sh",
"grpc_run_cpp_distribtest_test.sh",
])
genrule(

@ -17,8 +17,53 @@ Contains macros used for running bazelified tests.
"""
load(":dockerimage_current_versions.bzl", "DOCKERIMAGE_CURRENT_VERSIONS")
load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_rbe_exec_properties_dict")
def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None):
def _dockerized_sh_test(name, srcs = [], args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, docker_run_as_root = False, env = {}):
"""Runs sh_test under docker either via RBE or via docker sandbox."""
if docker_image_version:
image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None)
if not image_spec:
fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version)
else:
fail("docker_image_version attribute not set for dockerized test '%s'" % name)
exec_properties = create_rbe_exec_properties_dict(
labels = {
"workload": "misc",
"machine_size": "misc_large",
},
docker_network = "standard",
container_image = image_spec,
# TODO(jtattermusch): note that docker sandbox doesn't currently support "docker_run_as_root"
docker_run_as_root = docker_run_as_root,
)
# since the tests require special bazel args, only run them when explicitly requested
tags = ["manual"] + tags
# TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely
# to avoid running it outside of a docker container by accident.
test_args = {
"name": name,
"srcs": srcs,
"tags": tags,
"args": args,
"flaky": flaky,
"data": data,
"size": size,
"env": env,
"timeout": timeout,
"exec_compatible_with": exec_compatible_with,
"exec_properties": exec_properties,
}
native.sh_test(
**test_args
)
def grpc_run_tests_harness_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None):
"""Execute an run_tests.py-harness style test under bazel.
Args:
@ -45,7 +90,7 @@ def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout
] + args
srcs = [
"//tools/bazelify_tests:grpc_run_tests_py_test.sh",
"//tools/bazelify_tests:grpc_run_tests_harness_test.sh",
]
env = {}
@ -65,39 +110,66 @@ def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout
# TODO(jtattermusch): find a cleaner way to toggle ccache for builds.
env["GRPC_BUILD_ENABLE_CCACHE"] = "true"
# TODO(jtattermusch): use rbe_exec_properties helpers instead of manually specifying
# the properties, which is fragile.
exec_properties = {
"dockerNetwork": "standard", # TODO(jtattermusch): look into deactivating network for some actions
"label:workload": "misc", # always use a dedicated "misc" pool for running bazelified tests
"label:machine_size": "misc_large", # needed to override the default value of "small".
}
if docker_image_version:
image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None)
if not image_spec:
fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version)
exec_properties["container-image"] = image_spec
_dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env)
# since the tests require special bazel args, only run them when explicitly requested
tags = ["manual"] + tags
def grpc_run_bazel_distribtest_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None):
"""Execute bazel distribtest under bazel (an entire bazel build/test will run in a container as a single bazel action)
# TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely
# to avoid running it outside of a docker container by accident.
Args:
name: The name of the test.
args: The args to supply to the test binary.
data: Data dependencies.
size: The size of the test.
timeout: The test timeout.
tags: The tags for the test.
exec_compatible_with: A list of constraint values that must be
satisifed for the platform.
flaky: Whether this test is flaky.
docker_image_version: The docker .current_version file to use for docker containerization.
"""
test_args = {
"name": name,
"srcs": srcs,
"tags": tags,
"args": args,
"flaky": flaky,
"data": data,
"size": size,
"env": env,
"timeout": timeout,
"exec_compatible_with": exec_compatible_with,
"exec_properties": exec_properties,
}
data = [
"//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz",
] + data
native.sh_test(
**test_args
)
args = [
"$(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz)",
] + args
srcs = [
"//tools/bazelify_tests:grpc_run_bazel_distribtest_test.sh",
]
env = {}
_dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env)
def grpc_run_cpp_distribtest_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None):
"""Execute an C++ distribtest under bazel.
Args:
name: The name of the test.
args: The args to supply to the test binary.
data: Data dependencies.
size: The size of the test.
timeout: The test timeout.
tags: The tags for the test.
exec_compatible_with: A list of constraint values that must be
satisifed for the platform.
flaky: Whether this test is flaky.
docker_image_version: The docker .current_version file to use for docker containerization.
"""
data = [
"//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz",
] + data
args = [
"$(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz)",
] + args
srcs = [
"//tools/bazelify_tests:grpc_run_cpp_distribtest_test.sh",
]
# TODO(jtattermusch): revisit running docker as root (but currently some distribtests need to install stuff inside the docker container)
env = {}
_dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env, docker_run_as_root = True)

@ -0,0 +1,36 @@
#!/bin/bash
# Copyright 2023 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
ARCHIVE_WITH_SUBMODULES="$1"
shift
# The JUnit XML report file generated by run_tests.py is compatible with
# the report format accepted by bazel as the result for tests target.
REPORT_XML_FILE="${XML_OUTPUT_FILE}"
# Create report suite name from the last component of the bazel target's name.
REPORT_SUITE_NAME="$(echo ${TEST_TARGET} | sed 's|^.*[:/]||')"
# Extract grpc repo archive
tar -xopf ${ARCHIVE_WITH_SUBMODULES}
cd grpc
# Remove the override the "do not detect toolchain" setting that was set
# by the from .bazelrc configuration for the remote build.
# TODO(jtattermusch): find a better solution to avoid breaking toolchain detection.
export BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0
exec test/distrib/bazel/test_single_bazel_version.sh "$@"

@ -0,0 +1,34 @@
#!/bin/bash
# Copyright 2023 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
ARCHIVE_WITH_SUBMODULES="$1"
shift
# The JUnit XML report file generated by run_tests.py is compatible with
# the report format accepted by bazel as the result for tests target.
REPORT_XML_FILE="${XML_OUTPUT_FILE}"
# Create report suite name from the last component of the bazel target's name.
REPORT_SUITE_NAME="$(echo ${TEST_TARGET} | sed 's|^.*[:/]||')"
# Extract grpc repo archive
tar -xopf ${ARCHIVE_WITH_SUBMODULES}
cd grpc
# TODO(jtattermusch): adjust GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS for faster build.
# Run distribtest script passed as args
"$@"

@ -13,7 +13,7 @@
# limitations under the License.
load("//bazel:grpc_build_system.bzl", "grpc_package")
load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test")
load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_bazel_distribtest_test", "grpc_run_cpp_distribtest_test", "grpc_run_tests_harness_test")
load(":portability_tests.bzl", "generate_run_tests_portability_tests")
licenses(["notice"])
@ -23,7 +23,7 @@ grpc_package(name = "tools/bazelify_tests/test")
generate_run_tests_portability_tests(name = "portability_tests_linux")
# C/C++
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_c_linux_dbg",
size = "enormous",
args = [
@ -32,7 +32,7 @@ grpc_run_tests_py_test(
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version",
)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_c_linux_opt",
size = "enormous",
args = [
@ -41,7 +41,7 @@ grpc_run_tests_py_test(
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version",
)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_cpp_linux_dbg_build_only",
size = "enormous",
args = [
@ -50,7 +50,7 @@ grpc_run_tests_py_test(
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version",
)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_cpp_linux_opt_build_only",
size = "enormous",
args = [
@ -61,7 +61,7 @@ grpc_run_tests_py_test(
# TODO(jtattermusch): Reintroduce ruby tests once they pass.
# # Ruby
# grpc_run_tests_py_test(
# grpc_run_tests_harness_test(
# name = "runtests_ruby_linux_dbg",
# size = "enormous",
# args = [
@ -72,7 +72,7 @@ grpc_run_tests_py_test(
# use_login_shell = True, # ruby's docker image uses RVM which wierdly requires login shell
# )
# grpc_run_tests_py_test(
# grpc_run_tests_harness_test(
# name = "runtests_ruby_linux_opt",
# size = "enormous",
# args = [
@ -84,7 +84,7 @@ grpc_run_tests_py_test(
# )
# PHP
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_php_linux_dbg",
size = "enormous",
args = [
@ -93,7 +93,7 @@ grpc_run_tests_py_test(
docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version",
)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_php_linux_opt",
size = "enormous",
args = [
@ -104,7 +104,7 @@ grpc_run_tests_py_test(
# TODO(jtattermusch): Reintroduce python tests once they pass.
# # Python
# grpc_run_tests_py_test(
# grpc_run_tests_harness_test(
# name = "runtests_python_linux_opt",
# size = "enormous",
# args = [
@ -114,7 +114,7 @@ grpc_run_tests_py_test(
# )
# C#
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_csharp_linux_dbg",
size = "enormous",
args = [
@ -123,7 +123,7 @@ grpc_run_tests_py_test(
docker_image_version = "tools/dockerfile/test/csharp_debian11_x64.current_version",
)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_csharp_linux_opt",
size = "enormous",
args = [
@ -150,10 +150,100 @@ test_suite(
],
)
# TODO(jtattermusch): Reintroduce the test once it passes.
# grpc_run_cpp_distribtest_test(
# name = "cpp_distribtest_cmake_linux",
# size = "enormous",
# args = ["test/distrib/cpp/run_distrib_test_cmake.sh"],
# docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version",
# )
grpc_run_cpp_distribtest_test(
name = "cpp_distribtest_cmake_as_submodule_linux",
size = "enormous",
args = ["test/distrib/cpp/run_distrib_test_cmake_as_submodule.sh"],
docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version",
)
grpc_run_cpp_distribtest_test(
name = "cpp_distribtest_cmake_as_externalproject_linux",
size = "enormous",
args = ["test/distrib/cpp/run_distrib_test_cmake_as_externalproject.sh"],
docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version",
)
grpc_run_cpp_distribtest_test(
name = "cpp_distribtest_cmake_fetchcontent_linux",
size = "enormous",
args = ["test/distrib/cpp/run_distrib_test_cmake_fetchcontent.sh"],
docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version",
)
grpc_run_cpp_distribtest_test(
name = "cpp_distribtest_cmake_module_install_linux",
size = "enormous",
args = ["test/distrib/cpp/run_distrib_test_cmake_module_install.sh"],
docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version",
)
# TODO(jtattermusch): Reintroduce the test once it passes.
# grpc_run_cpp_distribtest_test(
# name = "cpp_distribtest_cmake_pkgconfig_linux",
# size = "enormous",
# args = ["test/distrib/cpp/run_distrib_test_cmake_pkgconfig.sh"],
# docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version",
# )
grpc_run_cpp_distribtest_test(
name = "cpp_distribtest_cmake_aarch64_cross_linux",
size = "enormous",
args = ["test/distrib/cpp/run_distrib_test_cmake_aarch64_cross.sh"],
docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_aarch64_cross_x64.current_version",
)
test_suite(
name = "cpp_distribtests_linux",
tests = [
":cpp_distribtest_cmake_aarch64_cross_linux",
":cpp_distribtest_cmake_as_externalproject_linux",
":cpp_distribtest_cmake_as_submodule_linux",
":cpp_distribtest_cmake_fetchcontent_linux",
#":cpp_distribtest_cmake_linux",
":cpp_distribtest_cmake_module_install_linux",
#":cpp_distribtest_cmake_pkgconfig_linux",
],
)
# TODO(jtattermusch): load supported versions from //bazel/supported_versions.txt
grpc_run_bazel_distribtest_test(
name = "bazel_distribtest_5.4.1",
size = "enormous",
args = ["5.4.1"],
docker_image_version = "tools/dockerfile/test/bazel.current_version",
)
# TODO(jtattermusch): load supported versions from //bazel/supported_versions.txt
grpc_run_bazel_distribtest_test(
name = "bazel_distribtest_6.1.2",
size = "enormous",
args = ["6.1.2"],
docker_image_version = "tools/dockerfile/test/bazel.current_version",
)
test_suite(
name = "bazel_distribtests_linux",
tests = [
":bazel_distribtest_5.4.1",
":bazel_distribtest_6.1.2",
],
)
test_suite(
name = "all_tests_linux",
tests = [
":basic_tests_linux",
":bazel_distribtests_linux",
":cpp_distribtests_linux",
":portability_tests_linux",
],
)

@ -16,7 +16,7 @@
Generates portability tests.
"""
load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test")
load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_harness_test")
def _safe_language_name(name):
"""Character '+' isn't allowed in bazel target name"""
@ -31,7 +31,7 @@ def generate_run_tests_portability_tests(name):
test_names = []
# portability C x86
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = "runtests_c_linux_dbg_x86",
args = ["-l c -c dbg"],
docker_image_version = "tools/dockerfile/test/cxx_debian11_x86.current_version",
@ -42,7 +42,7 @@ def generate_run_tests_portability_tests(name):
# C and C++ with no-exceptions on Linux
for language in ["c", "c++"]:
test_name = "runtests_%s_linux_dbg_noexcept_build_only" % _safe_language_name(language)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = test_name,
args = ["-l %s --config noexcept --build_only" % language],
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version",
@ -65,7 +65,7 @@ def generate_run_tests_portability_tests(name):
for compiler_name, args, docker_image_version in compiler_configs:
test_name = "runtests_%s_linux_dbg_%s_build_only" % (_safe_language_name(language), compiler_name)
grpc_run_tests_py_test(
grpc_run_tests_harness_test(
name = test_name,
args = ["-l %s -c dbg %s --build_only" % (language, args)],
docker_image_version = docker_image_version,
@ -75,7 +75,7 @@ def generate_run_tests_portability_tests(name):
# TODO(jtattermusch): Reintroduce the test once it passes.
# Python on alpine
#grpc_run_tests_py_test(
#grpc_run_tests_harness_test(
# name = "runtests_python_linux_dbg_alpine",
# args = [
# "-l python -c dbg --compiler python_alpine",

Loading…
Cancel
Save