From 5eb227936eb460f47de77b83fc08a0964268e10c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 10 Aug 2023 09:02:48 +0200 Subject: [PATCH] [bazel] Add more bazelified tests (#34005) - some cleanup - add bazel distribtests (to run the existing test_single_bazel_version.sh locally under a docker container created by RBE). - add c++ distribtests - rename `grpc_run_tests_py_test` macro to `grpc_run_tests_harness_test` - as requested previously to avoid confusion with `*_py_test` rules --- tools/bazelify_tests/BUILD | 5 +- tools/bazelify_tests/build_defs.bzl | 140 +++++++++++++----- .../grpc_run_bazel_distribtest_test.sh | 36 +++++ .../grpc_run_cpp_distribtest_test.sh | 34 +++++ ...test.sh => grpc_run_tests_harness_test.sh} | 0 tools/bazelify_tests/test/BUILD | 114 ++++++++++++-- .../bazelify_tests/test/portability_tests.bzl | 10 +- 7 files changed, 286 insertions(+), 53 deletions(-) create mode 100755 tools/bazelify_tests/grpc_run_bazel_distribtest_test.sh create mode 100755 tools/bazelify_tests/grpc_run_cpp_distribtest_test.sh rename tools/bazelify_tests/{grpc_run_tests_py_test.sh => grpc_run_tests_harness_test.sh} (100%) diff --git a/tools/bazelify_tests/BUILD b/tools/bazelify_tests/BUILD index a81d40f5b04..e2542760ca3 100644 --- a/tools/bazelify_tests/BUILD +++ b/tools/bazelify_tests/BUILD @@ -23,8 +23,9 @@ grpc_package( ) exports_files([ - "grpc_run_tests_py_test.sh", - "grpc_build_artifact_task.sh", + "grpc_run_tests_harness_test.sh", + "grpc_run_bazel_distribtest_test.sh", + "grpc_run_cpp_distribtest_test.sh", ]) genrule( diff --git a/tools/bazelify_tests/build_defs.bzl b/tools/bazelify_tests/build_defs.bzl index ec5054e6ddf..3a21773ba7e 100644 --- a/tools/bazelify_tests/build_defs.bzl +++ b/tools/bazelify_tests/build_defs.bzl @@ -17,8 +17,53 @@ Contains macros used for running bazelified tests. """ load(":dockerimage_current_versions.bzl", "DOCKERIMAGE_CURRENT_VERSIONS") +load("@bazel_toolchains//rules/exec_properties:exec_properties.bzl", "create_rbe_exec_properties_dict") -def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None): +def _dockerized_sh_test(name, srcs = [], args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, docker_run_as_root = False, env = {}): + """Runs sh_test under docker either via RBE or via docker sandbox.""" + if docker_image_version: + image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None) + if not image_spec: + fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version) + else: + fail("docker_image_version attribute not set for dockerized test '%s'" % name) + + exec_properties = create_rbe_exec_properties_dict( + labels = { + "workload": "misc", + "machine_size": "misc_large", + }, + docker_network = "standard", + container_image = image_spec, + # TODO(jtattermusch): note that docker sandbox doesn't currently support "docker_run_as_root" + docker_run_as_root = docker_run_as_root, + ) + + # since the tests require special bazel args, only run them when explicitly requested + tags = ["manual"] + tags + + # TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely + # to avoid running it outside of a docker container by accident. + + test_args = { + "name": name, + "srcs": srcs, + "tags": tags, + "args": args, + "flaky": flaky, + "data": data, + "size": size, + "env": env, + "timeout": timeout, + "exec_compatible_with": exec_compatible_with, + "exec_properties": exec_properties, + } + + native.sh_test( + **test_args + ) + +def grpc_run_tests_harness_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None): """Execute an run_tests.py-harness style test under bazel. Args: @@ -45,7 +90,7 @@ def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout ] + args srcs = [ - "//tools/bazelify_tests:grpc_run_tests_py_test.sh", + "//tools/bazelify_tests:grpc_run_tests_harness_test.sh", ] env = {} @@ -65,39 +110,66 @@ def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout # TODO(jtattermusch): find a cleaner way to toggle ccache for builds. env["GRPC_BUILD_ENABLE_CCACHE"] = "true" - # TODO(jtattermusch): use rbe_exec_properties helpers instead of manually specifying - # the properties, which is fragile. - exec_properties = { - "dockerNetwork": "standard", # TODO(jtattermusch): look into deactivating network for some actions - "label:workload": "misc", # always use a dedicated "misc" pool for running bazelified tests - "label:machine_size": "misc_large", # needed to override the default value of "small". - } - if docker_image_version: - image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None) - if not image_spec: - fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version) - exec_properties["container-image"] = image_spec + _dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env) - # since the tests require special bazel args, only run them when explicitly requested - tags = ["manual"] + tags +def grpc_run_bazel_distribtest_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None): + """Execute bazel distribtest under bazel (an entire bazel build/test will run in a container as a single bazel action) - # TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely - # to avoid running it outside of a docker container by accident. + Args: + name: The name of the test. + args: The args to supply to the test binary. + data: Data dependencies. + size: The size of the test. + timeout: The test timeout. + tags: The tags for the test. + exec_compatible_with: A list of constraint values that must be + satisifed for the platform. + flaky: Whether this test is flaky. + docker_image_version: The docker .current_version file to use for docker containerization. + """ - test_args = { - "name": name, - "srcs": srcs, - "tags": tags, - "args": args, - "flaky": flaky, - "data": data, - "size": size, - "env": env, - "timeout": timeout, - "exec_compatible_with": exec_compatible_with, - "exec_properties": exec_properties, - } + data = [ + "//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz", + ] + data - native.sh_test( - **test_args - ) + args = [ + "$(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz)", + ] + args + + srcs = [ + "//tools/bazelify_tests:grpc_run_bazel_distribtest_test.sh", + ] + env = {} + _dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env) + +def grpc_run_cpp_distribtest_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None): + """Execute an C++ distribtest under bazel. + + Args: + name: The name of the test. + args: The args to supply to the test binary. + data: Data dependencies. + size: The size of the test. + timeout: The test timeout. + tags: The tags for the test. + exec_compatible_with: A list of constraint values that must be + satisifed for the platform. + flaky: Whether this test is flaky. + docker_image_version: The docker .current_version file to use for docker containerization. + """ + + data = [ + "//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz", + ] + data + + args = [ + "$(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz)", + ] + args + + srcs = [ + "//tools/bazelify_tests:grpc_run_cpp_distribtest_test.sh", + ] + + # TODO(jtattermusch): revisit running docker as root (but currently some distribtests need to install stuff inside the docker container) + env = {} + _dockerized_sh_test(name = name, srcs = srcs, args = args, data = data, size = size, timeout = timeout, tags = tags, exec_compatible_with = exec_compatible_with, flaky = flaky, docker_image_version = docker_image_version, env = env, docker_run_as_root = True) diff --git a/tools/bazelify_tests/grpc_run_bazel_distribtest_test.sh b/tools/bazelify_tests/grpc_run_bazel_distribtest_test.sh new file mode 100755 index 00000000000..68e68211785 --- /dev/null +++ b/tools/bazelify_tests/grpc_run_bazel_distribtest_test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +ARCHIVE_WITH_SUBMODULES="$1" +shift + +# The JUnit XML report file generated by run_tests.py is compatible with +# the report format accepted by bazel as the result for tests target. +REPORT_XML_FILE="${XML_OUTPUT_FILE}" +# Create report suite name from the last component of the bazel target's name. +REPORT_SUITE_NAME="$(echo ${TEST_TARGET} | sed 's|^.*[:/]||')" + +# Extract grpc repo archive +tar -xopf ${ARCHIVE_WITH_SUBMODULES} +cd grpc + +# Remove the override the "do not detect toolchain" setting that was set +# by the from .bazelrc configuration for the remote build. +# TODO(jtattermusch): find a better solution to avoid breaking toolchain detection. +export BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0 + +exec test/distrib/bazel/test_single_bazel_version.sh "$@" diff --git a/tools/bazelify_tests/grpc_run_cpp_distribtest_test.sh b/tools/bazelify_tests/grpc_run_cpp_distribtest_test.sh new file mode 100755 index 00000000000..393fea8cf30 --- /dev/null +++ b/tools/bazelify_tests/grpc_run_cpp_distribtest_test.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +ARCHIVE_WITH_SUBMODULES="$1" +shift + +# The JUnit XML report file generated by run_tests.py is compatible with +# the report format accepted by bazel as the result for tests target. +REPORT_XML_FILE="${XML_OUTPUT_FILE}" +# Create report suite name from the last component of the bazel target's name. +REPORT_SUITE_NAME="$(echo ${TEST_TARGET} | sed 's|^.*[:/]||')" + +# Extract grpc repo archive +tar -xopf ${ARCHIVE_WITH_SUBMODULES} +cd grpc + +# TODO(jtattermusch): adjust GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS for faster build. + +# Run distribtest script passed as args +"$@" diff --git a/tools/bazelify_tests/grpc_run_tests_py_test.sh b/tools/bazelify_tests/grpc_run_tests_harness_test.sh similarity index 100% rename from tools/bazelify_tests/grpc_run_tests_py_test.sh rename to tools/bazelify_tests/grpc_run_tests_harness_test.sh diff --git a/tools/bazelify_tests/test/BUILD b/tools/bazelify_tests/test/BUILD index a6eac298bed..1d507208892 100644 --- a/tools/bazelify_tests/test/BUILD +++ b/tools/bazelify_tests/test/BUILD @@ -13,7 +13,7 @@ # limitations under the License. load("//bazel:grpc_build_system.bzl", "grpc_package") -load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") +load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_bazel_distribtest_test", "grpc_run_cpp_distribtest_test", "grpc_run_tests_harness_test") load(":portability_tests.bzl", "generate_run_tests_portability_tests") licenses(["notice"]) @@ -23,7 +23,7 @@ grpc_package(name = "tools/bazelify_tests/test") generate_run_tests_portability_tests(name = "portability_tests_linux") # C/C++ -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_c_linux_dbg", size = "enormous", args = [ @@ -32,7 +32,7 @@ grpc_run_tests_py_test( docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", ) -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_c_linux_opt", size = "enormous", args = [ @@ -41,7 +41,7 @@ grpc_run_tests_py_test( docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", ) -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_cpp_linux_dbg_build_only", size = "enormous", args = [ @@ -50,7 +50,7 @@ grpc_run_tests_py_test( docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", ) -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_cpp_linux_opt_build_only", size = "enormous", args = [ @@ -61,7 +61,7 @@ grpc_run_tests_py_test( # TODO(jtattermusch): Reintroduce ruby tests once they pass. # # Ruby -# grpc_run_tests_py_test( +# grpc_run_tests_harness_test( # name = "runtests_ruby_linux_dbg", # size = "enormous", # args = [ @@ -72,7 +72,7 @@ grpc_run_tests_py_test( # use_login_shell = True, # ruby's docker image uses RVM which wierdly requires login shell # ) -# grpc_run_tests_py_test( +# grpc_run_tests_harness_test( # name = "runtests_ruby_linux_opt", # size = "enormous", # args = [ @@ -84,7 +84,7 @@ grpc_run_tests_py_test( # ) # PHP -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_php_linux_dbg", size = "enormous", args = [ @@ -93,7 +93,7 @@ grpc_run_tests_py_test( docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version", ) -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_php_linux_opt", size = "enormous", args = [ @@ -104,7 +104,7 @@ grpc_run_tests_py_test( # TODO(jtattermusch): Reintroduce python tests once they pass. # # Python -# grpc_run_tests_py_test( +# grpc_run_tests_harness_test( # name = "runtests_python_linux_opt", # size = "enormous", # args = [ @@ -114,7 +114,7 @@ grpc_run_tests_py_test( # ) # C# -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_csharp_linux_dbg", size = "enormous", args = [ @@ -123,7 +123,7 @@ grpc_run_tests_py_test( docker_image_version = "tools/dockerfile/test/csharp_debian11_x64.current_version", ) -grpc_run_tests_py_test( +grpc_run_tests_harness_test( name = "runtests_csharp_linux_opt", size = "enormous", args = [ @@ -150,10 +150,100 @@ test_suite( ], ) +# TODO(jtattermusch): Reintroduce the test once it passes. +# grpc_run_cpp_distribtest_test( +# name = "cpp_distribtest_cmake_linux", +# size = "enormous", +# args = ["test/distrib/cpp/run_distrib_test_cmake.sh"], +# docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version", +# ) + +grpc_run_cpp_distribtest_test( + name = "cpp_distribtest_cmake_as_submodule_linux", + size = "enormous", + args = ["test/distrib/cpp/run_distrib_test_cmake_as_submodule.sh"], + docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version", +) + +grpc_run_cpp_distribtest_test( + name = "cpp_distribtest_cmake_as_externalproject_linux", + size = "enormous", + args = ["test/distrib/cpp/run_distrib_test_cmake_as_externalproject.sh"], + docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version", +) + +grpc_run_cpp_distribtest_test( + name = "cpp_distribtest_cmake_fetchcontent_linux", + size = "enormous", + args = ["test/distrib/cpp/run_distrib_test_cmake_fetchcontent.sh"], + docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version", +) + +grpc_run_cpp_distribtest_test( + name = "cpp_distribtest_cmake_module_install_linux", + size = "enormous", + args = ["test/distrib/cpp/run_distrib_test_cmake_module_install.sh"], + docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version", +) + +# TODO(jtattermusch): Reintroduce the test once it passes. +# grpc_run_cpp_distribtest_test( +# name = "cpp_distribtest_cmake_pkgconfig_linux", +# size = "enormous", +# args = ["test/distrib/cpp/run_distrib_test_cmake_pkgconfig.sh"], +# docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_x64.current_version", +# ) + +grpc_run_cpp_distribtest_test( + name = "cpp_distribtest_cmake_aarch64_cross_linux", + size = "enormous", + args = ["test/distrib/cpp/run_distrib_test_cmake_aarch64_cross.sh"], + docker_image_version = "tools/dockerfile/distribtest/cpp_debian10_aarch64_cross_x64.current_version", +) + +test_suite( + name = "cpp_distribtests_linux", + tests = [ + ":cpp_distribtest_cmake_aarch64_cross_linux", + ":cpp_distribtest_cmake_as_externalproject_linux", + ":cpp_distribtest_cmake_as_submodule_linux", + ":cpp_distribtest_cmake_fetchcontent_linux", + #":cpp_distribtest_cmake_linux", + ":cpp_distribtest_cmake_module_install_linux", + #":cpp_distribtest_cmake_pkgconfig_linux", + ], +) + +# TODO(jtattermusch): load supported versions from //bazel/supported_versions.txt +grpc_run_bazel_distribtest_test( + name = "bazel_distribtest_5.4.1", + size = "enormous", + args = ["5.4.1"], + docker_image_version = "tools/dockerfile/test/bazel.current_version", +) + +# TODO(jtattermusch): load supported versions from //bazel/supported_versions.txt +grpc_run_bazel_distribtest_test( + name = "bazel_distribtest_6.1.2", + size = "enormous", + args = ["6.1.2"], + docker_image_version = "tools/dockerfile/test/bazel.current_version", +) + +test_suite( + name = "bazel_distribtests_linux", + tests = [ + ":bazel_distribtest_5.4.1", + ":bazel_distribtest_6.1.2", + ], +) + test_suite( name = "all_tests_linux", tests = [ ":basic_tests_linux", + ":bazel_distribtests_linux", + ":cpp_distribtests_linux", ":portability_tests_linux", ], ) diff --git a/tools/bazelify_tests/test/portability_tests.bzl b/tools/bazelify_tests/test/portability_tests.bzl index 26afa9e5bea..b6e1e6680cd 100644 --- a/tools/bazelify_tests/test/portability_tests.bzl +++ b/tools/bazelify_tests/test/portability_tests.bzl @@ -16,7 +16,7 @@ Generates portability tests. """ -load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") +load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_harness_test") def _safe_language_name(name): """Character '+' isn't allowed in bazel target name""" @@ -31,7 +31,7 @@ def generate_run_tests_portability_tests(name): test_names = [] # portability C x86 - grpc_run_tests_py_test( + grpc_run_tests_harness_test( name = "runtests_c_linux_dbg_x86", args = ["-l c -c dbg"], docker_image_version = "tools/dockerfile/test/cxx_debian11_x86.current_version", @@ -42,7 +42,7 @@ def generate_run_tests_portability_tests(name): # C and C++ with no-exceptions on Linux for language in ["c", "c++"]: test_name = "runtests_%s_linux_dbg_noexcept_build_only" % _safe_language_name(language) - grpc_run_tests_py_test( + grpc_run_tests_harness_test( name = test_name, args = ["-l %s --config noexcept --build_only" % language], docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", @@ -65,7 +65,7 @@ def generate_run_tests_portability_tests(name): for compiler_name, args, docker_image_version in compiler_configs: test_name = "runtests_%s_linux_dbg_%s_build_only" % (_safe_language_name(language), compiler_name) - grpc_run_tests_py_test( + grpc_run_tests_harness_test( name = test_name, args = ["-l %s -c dbg %s --build_only" % (language, args)], docker_image_version = docker_image_version, @@ -75,7 +75,7 @@ def generate_run_tests_portability_tests(name): # TODO(jtattermusch): Reintroduce the test once it passes. # Python on alpine - #grpc_run_tests_py_test( + #grpc_run_tests_harness_test( # name = "runtests_python_linux_dbg_alpine", # args = [ # "-l python -c dbg --compiler python_alpine",