From 2b0a9d3916b3fd7787b35af46563552a37f9a3fb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 2 Aug 2023 14:15:19 +0200 Subject: [PATCH] [bazel] Teach bazel how to run non-bazel test suites under docker locally and on RBE (#33707) Add "bazelified" non-bazel tests. See tools/bazelify_tests/README.md for the core idea. - add a bunch of test targets that run under docker and execute tests that correspond to `run_tests.py -l LANG ...` - many more tests can be added in the future - to enable running some of the C/C++ portability tests easily, added support for `--cmake_extra_configure_args` in run_tests.py (the change is fairly small). Example passing build that shows how test results are structured: https://source.cloud.google.com/results/invocations/21295351-a3e3-4be1-b6e9-aaf52195a044/targets --- .../bazel/test_single_bazel_version.sh | 3 + tools/bazelify_tests/BUILD | 51 ++++++ tools/bazelify_tests/README.md | 45 +++++ tools/bazelify_tests/build_defs.bzl | 103 ++++++++++++ .../dockerimage_current_versions.bzl | 117 +++++++++++++ ...nerate_dockerimage_current_versions_bzl.sh | 81 +++++++++ tools/bazelify_tests/grpc_repo_archive.sh | 122 ++++++++++++++ .../bazelify_tests/grpc_run_tests_py_test.sh | 54 ++++++ tools/bazelify_tests/test/BUILD | 158 ++++++++++++++++++ .../bazelify_tests/test/portability_tests.bzl | 91 ++++++++++ tools/bazelify_tests/workspace_status_cmd.sh | 36 ++++ tools/dockerfile/push_testing_images.sh | 9 + .../linux/grpc_bazel_rbe_nonbazel.cfg | 52 ++++++ .../pull_request/grpc_bazel_rbe_nonbazel.cfg | 47 ++++++ .../include/rbe_base_config.bazelrc | 3 + .../remote_build/linux_docker_sandbox.bazelrc | 45 +++++ tools/run_tests/run_tests.py | 24 ++- 17 files changed, 1038 insertions(+), 3 deletions(-) create mode 100644 tools/bazelify_tests/BUILD create mode 100644 tools/bazelify_tests/README.md create mode 100644 tools/bazelify_tests/build_defs.bzl create mode 100644 tools/bazelify_tests/dockerimage_current_versions.bzl create mode 100755 tools/bazelify_tests/generate_dockerimage_current_versions_bzl.sh create mode 100755 tools/bazelify_tests/grpc_repo_archive.sh create mode 100755 tools/bazelify_tests/grpc_run_tests_py_test.sh create mode 100644 tools/bazelify_tests/test/BUILD create mode 100644 tools/bazelify_tests/test/portability_tests.bzl create mode 100755 tools/bazelify_tests/workspace_status_cmd.sh create mode 100644 tools/internal_ci/linux/grpc_bazel_rbe_nonbazel.cfg create mode 100644 tools/internal_ci/linux/pull_request/grpc_bazel_rbe_nonbazel.cfg create mode 100644 tools/remote_build/linux_docker_sandbox.bazelrc diff --git a/test/distrib/bazel/test_single_bazel_version.sh b/test/distrib/bazel/test_single_bazel_version.sh index 66e3240b8ab..6d01430598d 100755 --- a/test/distrib/bazel/test_single_bazel_version.sh +++ b/test/distrib/bazel/test_single_bazel_version.sh @@ -44,6 +44,9 @@ EXCLUDED_TARGETS=( # TODO(jtattermusch): add back once fixed "-//examples/android/binder/..." + + # Exclude bazelified tests as they contain some bazel hackery + "-//tools/bazelify_tests/..." ) TEST_DIRECTORIES=( diff --git a/tools/bazelify_tests/BUILD b/tools/bazelify_tests/BUILD new file mode 100644 index 00000000000..a81d40f5b04 --- /dev/null +++ b/tools/bazelify_tests/BUILD @@ -0,0 +1,51 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_package") + +licenses(["notice"]) + +# TODO(jtattermusch): adjust the visibility +grpc_package( + name = "tools/bazelify_tests", + visibility = "public", +) + +exports_files([ + "grpc_run_tests_py_test.sh", + "grpc_build_artifact_task.sh", +]) + +genrule( + name = "grpc_repo_archive", + srcs = + [ + "grpc_repo_archive.sh", + ], + outs = [ + "grpc_repo_archive.tar.gz", + "grpc_repo_archive_with_submodules.tar.gz", + ], + cmd = "$(location grpc_repo_archive.sh) $(location grpc_repo_archive.tar.gz) $(location grpc_repo_archive_with_submodules.tar.gz)", + local = True, + stamp = 1, + tags = [ + "manual", + ], +) + +alias( + name = "grpc_repo_archive_with_submodules", + actual = ":grpc_repo_archive", +) diff --git a/tools/bazelify_tests/README.md b/tools/bazelify_tests/README.md new file mode 100644 index 00000000000..fb14bbbf5a8 --- /dev/null +++ b/tools/bazelify_tests/README.md @@ -0,0 +1,45 @@ +# Non-Bazel native tests + +This directory contains logic that wraps builds and tests from the non-bazel realm to make them runnable under bazel. + +Examples: cmake builds, run_tests.py tests, artifacts, distribtests etc. + +NOTE: all tests and their setup under this directory are currently EXPERIMENTAL. + +## How it works + +The `//tools/bazelify_tests:repo_archive` target produces an archive that contains grpc at the current head with all its submodules. The rule uses a few tricks to achieve this: + +* Uses a workspace status command to obtain the commit SHAs of grpc and all submodules from the workspace. +* When running, it actually jailbreaks from the bazel execroot to access the bazel workspace and create the necessary archives. +* The produced archives are deterministic (they have the same checksum if neither grpc or its submodules have changed). +* The target is defined in such a way so that it behaves "reasonably" from bazel's perspective (always re-runs if commit SHAs have changed, can be cached if not). + +After grpc source code is archived, the "bazelified" tests basically depend on the archive and they run a script under a docker container. +The script unpacks the archived grpc code and creates a temporary workspace (under bazel's target execroot) and then performs the actions that are needed +(e.g. run the run_tests.py test harness, run cmake build etc). + +There are two ways the test targets can run under a docker container: + +* When running on RBE, all actions run under a docker container by definition. +* When running locally, bazel will start a docker container for each action when [docker sandbox](https://bazel.build/remote/sandbox) is used. + (Note that the docker sandbox currently [doesn't work on windows](https://github.com/bazelbuild/bazel/issues/19101)) + +In both cases, the docker image which is used for any given action is determined by the action's `exec_properties` and can be specified as a default +(e.g. by RBE toolchain or by setting `--experimental_docker_image` flag) or explicitly for each action. For most tests in this directory, +the test rules actually configure the `exec_properties` for you, based on selecting one of the gRPC's testing docker images. + +## Run tests on RBE + +``` +# "--genrule_strategy=remote,local" allows fallback to local execution if action doesn't support running remotely +# (required to be able to run the //tools/bazelify_tests:repo_archive target). +tools/bazel --bazelrc=tools/remote_build/linux.bazelrc test --genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh //tools/bazelify_tests/test:basic_tests_linux +``` + +## Run tests locally under bazel's docker sandbox + +``` +tools/bazel --bazelrc=tools/remote_build/linux_docker_sandbox.bazelrc test --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh //tools/bazelify_tests/test:basic_tests_linux +``` + diff --git a/tools/bazelify_tests/build_defs.bzl b/tools/bazelify_tests/build_defs.bzl new file mode 100644 index 00000000000..ec5054e6ddf --- /dev/null +++ b/tools/bazelify_tests/build_defs.bzl @@ -0,0 +1,103 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Contains macros used for running bazelified tests. +""" + +load(":dockerimage_current_versions.bzl", "DOCKERIMAGE_CURRENT_VERSIONS") + +def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None): + """Execute an run_tests.py-harness style test under bazel. + + Args: + name: The name of the test. + args: The args to supply to the test binary. + data: Data dependencies. + size: The size of the test. + timeout: The test timeout. + tags: The tags for the test. + exec_compatible_with: A list of constraint values that must be + satisifed for the platform. + flaky: Whether this test is flaky. + docker_image_version: The docker .current_version file to use for docker containerization. + use_login_shell: If True, the run_tests.py command will run under a login shell. + prepare_script: Optional script that will be sourced before run_tests.py runs. + """ + + data = [ + "//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz", + ] + data + + args = [ + "$(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz)", + ] + args + + srcs = [ + "//tools/bazelify_tests:grpc_run_tests_py_test.sh", + ] + + env = {} + + if use_login_shell: + env["GRPC_RUNTESTS_USE_LOGIN_SHELL"] = "1" + + if prepare_script: + data = data + [prepare_script] + env["GRPC_RUNTESTS_PREPARE_SCRIPT"] = "$(location " + prepare_script + ")" + + # Enable ccache by default. This is important for speeding up the C++ cmake build, + # which isn't very efficient and tends to recompile some source files multiple times. + # Even though only the local disk cache is enabled (local to the docker container, + # so will be thrown away after the bazel actions finishes), ccache still speeds up + # the C++ build significantly. + # TODO(jtattermusch): find a cleaner way to toggle ccache for builds. + env["GRPC_BUILD_ENABLE_CCACHE"] = "true" + + # TODO(jtattermusch): use rbe_exec_properties helpers instead of manually specifying + # the properties, which is fragile. + exec_properties = { + "dockerNetwork": "standard", # TODO(jtattermusch): look into deactivating network for some actions + "label:workload": "misc", # always use a dedicated "misc" pool for running bazelified tests + "label:machine_size": "misc_large", # needed to override the default value of "small". + } + if docker_image_version: + image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None) + if not image_spec: + fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version) + exec_properties["container-image"] = image_spec + + # since the tests require special bazel args, only run them when explicitly requested + tags = ["manual"] + tags + + # TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely + # to avoid running it outside of a docker container by accident. + + test_args = { + "name": name, + "srcs": srcs, + "tags": tags, + "args": args, + "flaky": flaky, + "data": data, + "size": size, + "env": env, + "timeout": timeout, + "exec_compatible_with": exec_compatible_with, + "exec_properties": exec_properties, + } + + native.sh_test( + **test_args + ) diff --git a/tools/bazelify_tests/dockerimage_current_versions.bzl b/tools/bazelify_tests/dockerimage_current_versions.bzl new file mode 100644 index 00000000000..79147f8ea08 --- /dev/null +++ b/tools/bazelify_tests/dockerimage_current_versions.bzl @@ -0,0 +1,117 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file is generated by generate_dockerimage_current_versions_bzl.sh +It makes the info from testing docker image *.current_version files +accessible to bazel builds. +""" + +DOCKERIMAGE_CURRENT_VERSIONS = { + "third_party/rake-compiler-dock/rake_aarch64-linux.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_aarch64-linux@sha256:61a46ab67972990aea77024817d29ca6fa43d2639fe4aaef9c30e031f84519a9", + "third_party/rake-compiler-dock/rake_arm64-darwin.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_arm64-darwin@sha256:e0eb1f9f632fb18d4f244b7297d1a5e7cf60ae58e649ac5b2f8ac6266ea07128", + "third_party/rake-compiler-dock/rake_x64-mingw-ucrt.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw-ucrt@sha256:63490b0000c6011a19983fef637efc69a2ae0f67b7a4e29cd36db53c881e908d", + "third_party/rake-compiler-dock/rake_x64-mingw32.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw32@sha256:63ece6e9b336b7cbf66eaa0201505b0579ac06cd7802f19b44c3a816d5617c17", + "third_party/rake-compiler-dock/rake_x86-linux.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-linux@sha256:71e3afca0843bf7bd5da7fa04bff40ad976e76aa5867936166b30251d0a692d8", + "third_party/rake-compiler-dock/rake_x86-mingw32.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-mingw32@sha256:629be8f57e2d50b123584f2cfa00ff5b968cc2cc3b2a6b874acd07100a3eb96d", + "third_party/rake-compiler-dock/rake_x86_64-darwin.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-darwin@sha256:8dd11cad778d9fc01c3555a57254016f5db7227309d24f50a192a6db80d4a51c", + "third_party/rake-compiler-dock/rake_x86_64-linux.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-linux@sha256:9aa77587fa4d4c25c91d0ccca8eb806cf0738a4b67eb4b54d40324185658194e", + "tools/dockerfile/distribtest/cpp_debian10_aarch64_cross_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cpp_debian10_aarch64_cross_x64@sha256:15eeafcd816cb32a0d44da22f654749352a92fec9626dc028b39948897d5bea3", + "tools/dockerfile/distribtest/cpp_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cpp_debian10_x64@sha256:904e3db8521697768f94aa08230063b474246184e126f74a41b98a6f4aaf6a49", + "tools/dockerfile/distribtest/csharp_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_alpine_x64@sha256:d018105349fcabdc3aa0649c1381d840c613df6b442a53a751d7dc839a80d429", + "tools/dockerfile/distribtest/csharp_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_centos7_x64@sha256:ec715dd5fbd621789e7598c8d4ac346a7b4037b0cc83fbb29990dc8e4c1f1a13", + "tools/dockerfile/distribtest/csharp_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian10_x64@sha256:8c3838e731da70566adc6f989f2c29351fdb2f629e8797928699fff24b3a0938", + "tools/dockerfile/distribtest/csharp_dotnet31_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet31_x64@sha256:fee52df6064ff84bc9af644c2ea17ab579de3401e3a167d0d43383c24f0d500f", + "tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:408425cd74bb8b79a3b09a64ea6c54f6cdc0e757a3469f31effc017a7187e442", + "tools/dockerfile/distribtest/csharp_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu1604_x64@sha256:e0f44406df14a28ce0a0f4e26c74c95f0fa5dddadf1fdbb2a3793b7c8ef8fa63", + "tools/dockerfile/distribtest/php7_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian10_x64@sha256:e760a60f2dce2dada571d9b07447a9f99ffeeb366a309dbbb5dc0a43991c22dc", + "tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:699ac7b86199406fa27e88f30a1c623ef34ac33f6d9330fd13a6f6457ee4e19f", + "tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1", + "tools/dockerfile/distribtest/python_buster_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_buster_x64@sha256:e501dc8e2f4ab9cd4382974759a879a27c065c8fed5327f538764298fc5c4972", + "tools/dockerfile/distribtest/python_buster_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_buster_x86@sha256:185fbb174525d67b6146f4d233c804c589b0b57d783bb1bf95bc47cfe792754e", + "tools/dockerfile/distribtest/python_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_centos7_x64@sha256:39afaa687cb8516eef1621ed789326fdde2014fd3c81d11a1ded72f2d5285fe1", + "tools/dockerfile/distribtest/python_dev_alpine3.7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_alpine3.7_x64@sha256:7c08f67211a49eb72ad08c29de5c64a914c066d9c1670b712e717571b8d5c7e2", + "tools/dockerfile/distribtest/python_dev_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_arch_x64@sha256:29f179ef2083ee6addd57e90f58781fdc1cb5dc3dd3e228da1af38785b921f35", + "tools/dockerfile/distribtest/python_dev_buster_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_buster_x64@sha256:e30d6efdeac24e5136cc169d503a239df22147bfb121d27feb1f87d58a8fe64e", + "tools/dockerfile/distribtest/python_dev_buster_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_buster_x86@sha256:179146fd5d5cc15846c6bf0284c2e261f383caf944559d2d9f7a5af0e0f7152d", + "tools/dockerfile/distribtest/python_dev_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_centos7_x64@sha256:e6e9a1b23a0a543050db91e17d621aa899bad04308adaf961c11fa88ba941a95", + "tools/dockerfile/distribtest/python_dev_fedora34_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_fedora34_x64@sha256:20bc3a6283a99407eb637b3cde1ff3e1288a1e21388a1dc385c2b4df5a1eb1c2", + "tools/dockerfile/distribtest/python_dev_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_ubuntu1604_x64@sha256:167134c3a43e7d2608c893cc98a5066eae93c6af97ef5a1e69d643cbc7fefc43", + "tools/dockerfile/distribtest/python_dev_ubuntu1804_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_ubuntu1804_x64@sha256:157a89cd6d0e69b89ac1975e0314aade556a35aafbaa5fe9f9890f90321d6c89", + "tools/dockerfile/distribtest/python_dev_ubuntu2004_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_ubuntu2004_x64@sha256:91f0d88c43ec52ecd63f99acb424c88ff9a67fa046fae207a75e99bee37eef11", + "tools/dockerfile/distribtest/python_fedora34_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_fedora34_x64@sha256:5aa8e41d627ddd6bc10aae6b12d25ded90ba8554a63b279f43f44e0d6cf001dd", + "tools/dockerfile/distribtest/python_opensuse_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_opensuse_x64@sha256:da52566b078d10e537aa219e59641731a57e5dc7d17d6737f5e5a7d447acf5cc", + "tools/dockerfile/distribtest/python_python38_buster_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_python38_buster_aarch64@sha256:487b9af2ad1459ee2630694e61074d4ac525d4f90b2bdb026dbf6f77fb3e9878", + "tools/dockerfile/distribtest/python_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu1604_x64@sha256:44a821a9f5431122c2e239ba35cf181c2fde84a5d866e8add338599565881492", + "tools/dockerfile/distribtest/python_ubuntu1804_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu1804_x64@sha256:edcd5f342d77ad9129cc0a0e6988b47b144815e7a93091d5b45e850111eefbcf", + "tools/dockerfile/distribtest/python_ubuntu2004_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu2004_x64@sha256:342e9dc23b674ad256b220745745be818708a1baa25a2690f0d4f777e28a22a3", + "tools/dockerfile/distribtest/ruby_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_centos7_x64@sha256:4d529b984b78ca179086f7f9b416605e2d9a96ca0a28a71f4421bb5ffdc18f96", + "tools/dockerfile/distribtest/ruby_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64@sha256:1298c39c950b2a48261555b6cff1ae66230a5020f100d3b381759285f0caf84e", + "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_6.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_2_6@sha256:3ef9a84a77276f2ccaca3c97336573f76ed02719645e4bb67ccdf1f33da99fc8", + "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_2_7@sha256:5ee26ad3abe2683c9a8ee03987ab0ae63f50793c3d3f5e4be6e6cbacb4556fcf", + "tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_0@sha256:9190da90a2a95eca1370cef64dcba7ddee9f59cc7487093da6711c1280a0b0f9", + "tools/dockerfile/distribtest/ruby_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_ubuntu1604_x64@sha256:e0276968184a6c1e16de4e6afbbd469df91b27e40d061340841c76e864fdcb50", + "tools/dockerfile/distribtest/ruby_ubuntu1804_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_ubuntu1804_x64@sha256:d38b3dd34cffc027e9a1bf82bc7ace75b8a9829c2d04d5cf7cc8323655edd27a", + "tools/dockerfile/grpc_artifact_centos6_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x64@sha256:3285047265ea2b7c5d4df4c769b2d05f56288d947c75e16d27ae2dee693f791b", + "tools/dockerfile/grpc_artifact_centos6_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x86@sha256:19783239da92208f0f39cf563529cd02e889920497ef81c60d20391fa998af62", + "tools/dockerfile/grpc_artifact_protoc_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_protoc_aarch64@sha256:1a3957f32e81259e6f3c602bd67feb132ebc5a5f23e9cb0bf63ba34b91185982", + "tools/dockerfile/grpc_artifact_python_linux_armv7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_linux_armv7@sha256:4f817dece74bbdc7c4fccdc9b0a25cefb9101781a60bf0bb827e533e79f9b1f2", + "tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:d56ea4394ea5ea9d09f940d1dba31e6196a8e919f60c6a4966a9192faa997f11", + "tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:67ab746e6da576606ebf41ad81027ad897544445fb93d5d5ca5f9d9b5428ec84", + "tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:993a963ac3985f8634951e1573d34e24b3868dfff3ad4ae4875dd2c47b73224f", + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:09bf18cc793d55cfc48d8e88b8b6e6914e9df2b35ec417fe77a4e20bfa251df7", + "tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:0512449e7d218c7687eb447701c8c6a33153a722722b76b2423ec58440a027de", + "tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878", + "tools/dockerfile/interoptest/grpc_interop_cxx.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_cxx@sha256:a69a1ed729137c3ea347f0a3488524573285be7832dd74cec830db57b61a9b8c", + "tools/dockerfile/interoptest/grpc_interop_dart.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_dart@sha256:5e335005b27709f0882c5723affafa55094bd27a0cda7ce91c718deed157f2bb", + "tools/dockerfile/interoptest/grpc_interop_go.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go@sha256:889e7ff34399a5e16af87940d1eaa239e56da307f7faca3f8f1d28379c2e3df3", + "tools/dockerfile/interoptest/grpc_interop_go1.11.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.11@sha256:29cde59287843a3208c0cabeaf430cf813846a738c8a1b9692e68b54bbbdcc2d", + "tools/dockerfile/interoptest/grpc_interop_go1.16.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.16@sha256:d5b2b0c02e7a8196fea704196a8221994983c22eece2ac2324e095e8972a957f", + "tools/dockerfile/interoptest/grpc_interop_go1.19.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.19@sha256:889e7ff34399a5e16af87940d1eaa239e56da307f7faca3f8f1d28379c2e3df3", + "tools/dockerfile/interoptest/grpc_interop_go1.8.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.8@sha256:7830a301b37539252c592b9cd7fa30a6142d0afc717a05fc8d2b82c74fb45efe", + "tools/dockerfile/interoptest/grpc_interop_http2.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_http2@sha256:e3f247d8038374848fadf7215b841e3575c0b2a4217feb503a79b8004b164c5a", + "tools/dockerfile/interoptest/grpc_interop_java.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_java@sha256:d9210764071662ba2f377dafcaff4b743f41e4dff1876dd47df7b1c6950f88af", + "tools/dockerfile/interoptest/grpc_interop_node.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_node@sha256:337e9995563e4f569b4daf843d0a2af0619e086481ce3ba3f888434eb2ddc28b", + "tools/dockerfile/interoptest/grpc_interop_nodepurejs.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_nodepurejs@sha256:9061077a17eb6f2306af563ed85c12630480f6d6ce15919d67ef5567dbab559e", + "tools/dockerfile/interoptest/grpc_interop_php7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_php7@sha256:09f4b895117c81506c423360b617917d06d3f7f0b78e4cca25eaec547ba6991e", + "tools/dockerfile/interoptest/grpc_interop_python.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_python@sha256:fef1247f8256be2b9841331e7d21b0046da21a4a6d34a62addb36f62124725cf", + "tools/dockerfile/interoptest/grpc_interop_pythonasyncio.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_pythonasyncio@sha256:bd4cdc8a71ef339193e178ce20d2b47a0b2aa25fc86c0b5740b9d86a2d4a0caa", + "tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:0de52450b29cf91365e623b020cd97722c307510ba1813bee09264e0a49acdbc", + "tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52", + "tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:2bddf36ae504968b35f97e4a6c9b74864473689e84049675c30afb70f868d897", + "tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:1118150d9d9479787165fff49f660a3dc633f1c57604305460172fc1916aa022", + "tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:dd4a71e2852faf24b45e21445331f400bb278c83f6ad6d7d89d43c9ac564d529", + "tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:85341f035e44601a93d16ff5b9b5810a0da313af03e2a76cf4135144633e0bab", + "tools/dockerfile/test/csharp_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_arm64@sha256:e1345c81aaab856eab0635ddbe612294bee9ee38d4938d3434eab277de6029b8", + "tools/dockerfile/test/csharp_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_x64@sha256:a38ffe41c25486ad0624f54b4a5fa11e74772a06f537c553b3ae3944511ef348", + "tools/dockerfile/test/cxx_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_alpine_x64@sha256:61fc7408e1171d9470bdd6920cc9da34e31fc43115b80f0fb6f7b9669ba6e366", + "tools/dockerfile/test/cxx_clang_15_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_clang_15_x64@sha256:aaac47bdeccfcf43331963a75df6a377923c69d1b57ea076c2072b140e00af65", + "tools/dockerfile/test/cxx_clang_6_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_clang_6_x64@sha256:79ecf682702190564c41289ffe00d4e6f80c104807cca324340349e84288ad99", + "tools/dockerfile/test/cxx_debian11_openssl102_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian11_openssl102_x64@sha256:8552c41ecca59e32cb3079981cce0b2993a443f1730562a7f19a172ab29f1b2d", + "tools/dockerfile/test/cxx_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian11_x64@sha256:f4d2b360e8a49d95e8e92971566674a06015427c2488a841b3386feb41d2ff22", + "tools/dockerfile/test/cxx_debian11_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian11_x86@sha256:77a0be06797567ad9e4924bb5f1a523cd23555af0518a1525fc4a940d60d035c", + "tools/dockerfile/test/cxx_debian12_openssl309_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian12_openssl309_x64@sha256:e8878bf31f42f8af6a7f3b45d0cb79f29fc46c44721b4a8357a2070ab1f5b3e0", + "tools/dockerfile/test/cxx_gcc_12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_gcc_12_x64@sha256:ca86af6cb592b4426585a67c7fe58d9925a6e5413801ab45831cd268102c4211", + "tools/dockerfile/test/cxx_gcc_7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_gcc_7_x64@sha256:7d1af94c7329b6b09f6266a56380c0690a31e9121abc89cb8a57820e0f6eb3bb", + "tools/dockerfile/test/php73_zts_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php73_zts_debian11_x64@sha256:db46a738bf187ffcabbd278a716930c87f90dec2599349c5a52937f9a17e96f8", + "tools/dockerfile/test/php7_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_arm64@sha256:444e25f9e3a89c2438e4d5e6f3904c5a1f4d1fb961f8456333ebe3e36e301a4e", + "tools/dockerfile/test/php7_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_x64@sha256:018d422abf144fc93e9027fd994f7d6aab453fffbe4a669d622dd3d1c1fe9ee7", + "tools/dockerfile/test/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:d10159225ae25276b7ae7bfc4230150e4b0a8ce7be833d904bdd4ecdfdc91c6e", + "tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:868cfb50e465f086b75bb65a7fab6d15b1edefabcd8c1826340acefb6ea1737f", + "tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:4f29e539941d22b7abb911f9b6b3101ff5c7c4fb75585bfe3b7389251ea6be1d", + "tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8", + "tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:9503d80a40555aba4dd531b64354ad8036c6b37e162c93e7994ca23d89bc7d41", + "tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:3f01369c3e5707fa63007820b30461b9e32a4b729d81cb92d19669d7966a8584", + "tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:189e07d8503aa15344e3c8f565783659c3e2edc5b8ca455ec427de1e29ef4504", +} diff --git a/tools/bazelify_tests/generate_dockerimage_current_versions_bzl.sh b/tools/bazelify_tests/generate_dockerimage_current_versions_bzl.sh new file mode 100755 index 00000000000..ab5eee8923b --- /dev/null +++ b/tools/bazelify_tests/generate_dockerimage_current_versions_bzl.sh @@ -0,0 +1,81 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generates dockerimage_current_versions.bzl file from the .current_version +# files found throughout the repository. + +set -e + +cd $(dirname $0)/../.. + +OUTPUT_FILE=tools/bazelify_tests/dockerimage_current_versions.bzl +if [ "${CHECK_MODE}" != "" ] +then + # generate into temporary file instead + ORIG_FILE="${OUTPUT_FILE}" + OUTPUT_FILE="$(mktemp)" +fi + +# Generate license header and module docstring. +cat >"${OUTPUT_FILE}" << EOF +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This file is generated by generate_dockerimage_current_versions_bzl.sh +It makes the info from testing docker image *.current_version files +accessible to bazel builds. +""" + +EOF + +echo "DOCKERIMAGE_CURRENT_VERSIONS = {" >>"${OUTPUT_FILE}" + +for current_version_file in $(find tools/ third_party/rake-compiler-dock -name '*.current_version' | LC_ALL=C sort) +do + # Remove the docker image tag and only keep the SHA256 digest for the dockerimage. + # For some reason, bazel's "container-image" exec property doesn't accept image spec + # with both tag and SHA256 digest. + echo " \"${current_version_file}\": \"docker://$(cat ${current_version_file} | sed 's/:[a-f0-9]*@sha256/@sha256/')\"," >>"${OUTPUT_FILE}" +done + +echo "}" >>"${OUTPUT_FILE}" + +if [ "${CHECK_MODE}" != "" ] +then + echo "Checking that ${ORIG_FILE} is up-to-date." + diff "${OUTPUT_FILE}" "${ORIG_FILE}" || CHECK_FAILED=true + + if [ "${CHECK_FAILED}" != "" ] + then + echo "CHECK FAILED: Generated file ${ORIG_FILE} is out of date and needs to be regenerated." + exit 1 + else + echo "${ORIG_FILE} is up-to-date." + fi +else + echo "Generated ${OUTPUT_FILE}" +fi diff --git a/tools/bazelify_tests/grpc_repo_archive.sh b/tools/bazelify_tests/grpc_repo_archive.sh new file mode 100755 index 00000000000..2f716ebe80d --- /dev/null +++ b/tools/bazelify_tests/grpc_repo_archive.sh @@ -0,0 +1,122 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +ARCHIVE_WO_SUBMODULES="$1" +ARCHIVE_WITH_SUBMODULES="$2" +export ARCHIVES_DIR="$(pwd)/archives" + +export ARCHIVE_FORMAT=tar + +mkdir -p "${ARCHIVES_DIR}" +rm -rf "${ARCHIVES_DIR}"/* + +# TODO(jtattermusch): This script is currently only tested on linux. +# Nothing prevents it from working on other systems in principle, +# but more work is needed. + +# HACK: To be able to collect all grpc source files as an archive +# we need to break from bazel's "sandbox" to be able to read files +# from the original bazel workspace (which in our case is the grpc repository root) +# This action runs with the "standalone" (a.k.a) local strategy, +# so path to the original bazel workspace from where this was invoked +# can be obtained by resolving the link that points to one of the +# source files. +# 1. find first component of the relative path to this script +# 2. resolve the symlink (it will point to same dir in the workspace) +# 3. one level up is the root of the original bazel workspace +FIRST_PATH_COMPONENT="$(echo $0 | sed 's|/.*||')" +ORIGINAL_BAZEL_WORKSPACE_ROOT="$(dirname $(readlink ${FIRST_PATH_COMPONENT}))" + +# extract STABLE_GIT_COMMIT from stable-status.txt +GRPC_GIT_COMMIT_FROM_STABLE_STATUS=$(grep ^STABLE_GRPC_GIT_COMMIT bazel-out/stable-status.txt | cut -d' ' -f2) + +if [ "${GRPC_GIT_COMMIT_FROM_STABLE_STATUS}" == "" ] +then + echo "Failed to obtain info about gRPC git commit from the bazel workspace. Make sure you invoke bazel with --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh" >&2 + exit 1 +fi + +GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS=$(grep ^STABLE_GRPC_UNCOMMITED_PATCH_CHECKSUM bazel-out/stable-status.txt | cut -d' ' -f2) +GRPC_GIT_WORKSPACE_DIRTY_FROM_STABLE_STATUS=$(grep ^STABLE_GRPC_GIT_WORKSPACE_DIRTY bazel-out/stable-status.txt | cut -d' ' -f2) + +pushd ${ORIGINAL_BAZEL_WORKSPACE_ROOT} >/dev/null + +if [ "${GRPC_GIT_COMMIT_FROM_STABLE_STATUS}" != "$(git rev-parse HEAD)" ] +then + echo "Info about gRPC commit from stable-status.txt doesn't match the commit at which the workspace root is." >&2 + echo "This is unexpected and giving up early is better than risking to end up with bogus results." >&2 + exit 1 +fi + +mkdir -p ${ARCHIVES_DIR}/grpc +git archive --format="${ARCHIVE_FORMAT}" HEAD >"${ARCHIVES_DIR}/grpc/$(git rev-parse HEAD).${ARCHIVE_FORMAT}" + +if [ "${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}" != "" ] +then + git diff HEAD >"${ARCHIVES_DIR}/grpc/grpc_uncommited_${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}.patch" + # check that the actual checksum of the patch file is what we expect it to be + echo "${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS} ${ARCHIVES_DIR}/grpc/grpc_uncommited_${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}.patch" | sha256sum --quiet --check +fi + +# produce archive for each submodule +git submodule --quiet foreach 'git_commit="$(git rev-parse HEAD)"; mkdir -p ${ARCHIVES_DIR}/${name}; git archive --format=${ARCHIVE_FORMAT} HEAD >${ARCHIVES_DIR}/${name}/${git_commit}.${ARCHIVE_FORMAT}' + +popd >/dev/null + +# Extract grpc +mkdir grpc +tar -xopf "${ARCHIVES_DIR}/grpc/${GRPC_GIT_COMMIT_FROM_STABLE_STATUS}.${ARCHIVE_FORMAT}" -C grpc + +# apply the patch +if [ "${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}" != "" ] +then + pushd grpc >/dev/null + patch --quiet -p1 <"${ARCHIVES_DIR}/grpc/grpc_uncommited_${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}.patch" + popd >/dev/null +fi + +# The archive produced need to be deterministic/stable. +# Passing the following args to tar should be enough to make them so. +# See https://reproducible-builds.org/docs/archives/ +DETERMINISTIC_TAR_ARGS=( + --sort=name + # use a fixed mtime timestamp for all files (2015-01-01 00:00Z works just fine) + --mtime="@1420070400" + --owner=0 + --group=0 + --numeric-owner + --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime +) + +# create archive without submodules first +tar "${DETERMINISTIC_TAR_ARGS[@]}" -czf ${ARCHIVE_WO_SUBMODULES} grpc + +SUBMODULE_ARCHIVE_LIST="$(grep 'STABLE_GRPC_SUBMODULE_ARCHIVES ' bazel-out/stable-status.txt | sed 's/^STABLE_GRPC_SUBMODULE_ARCHIVES //')" +# TODO(jtattermusch): handle spaces in submodule directory path +for submodule_archive in ${SUBMODULE_ARCHIVE_LIST} +do + archive_subdir="grpc/$(dirname ${submodule_archive})" + mkdir -p $archive_subdir + # Extract submodule archive under the correct subdirectory in grpc + tar -xopf "${ARCHIVES_DIR}/${submodule_archive}.${ARCHIVE_FORMAT}" -C $archive_subdir +done + +# create archive with everything +tar "${DETERMINISTIC_TAR_ARGS[@]}" -czf ${ARCHIVE_WITH_SUBMODULES} grpc + +# Cleanup intermediate files we created +rm -rf "${ARCHIVES_DIR}" grpc diff --git a/tools/bazelify_tests/grpc_run_tests_py_test.sh b/tools/bazelify_tests/grpc_run_tests_py_test.sh new file mode 100755 index 00000000000..695bcaf4601 --- /dev/null +++ b/tools/bazelify_tests/grpc_run_tests_py_test.sh @@ -0,0 +1,54 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +if [ "${GRPC_RUNTESTS_USE_LOGIN_SHELL}" != "" ] +then + unset GRPC_RUNTESTS_USE_LOGIN_SHELL + # respawn the entire script using login shell + exec bash -l "$0" "$@" +fi + +ARCHIVE_WITH_SUBMODULES="$1" +shift + +# The JUnit XML report file generated by run_tests.py is compatible with +# the report format accepted by bazel as the result for tests target. +REPORT_XML_FILE="${XML_OUTPUT_FILE}" +# Create report suite name from the last component of the bazel target's name. +REPORT_SUITE_NAME="$(echo ${TEST_TARGET} | sed 's|^.*[:/]||')" + +# Extract grpc repo archive +tar -xopf ${ARCHIVE_WITH_SUBMODULES} +cd grpc + +if [ "${GRPC_RUNTESTS_PREPARE_SCRIPT}" != "" ] +then + source "../${GRPC_RUNTESTS_PREPARE_SCRIPT}" +fi + +python3 tools/run_tests/run_tests.py -t -j "$(nproc)" -x "${REPORT_XML_FILE}" --report_suite_name "${REPORT_SUITE_NAME}" "$@" || FAILED="true" + +if [ -x "$(command -v ccache)" ] +then + ccache --show-stats || true +fi + +if [ "$FAILED" != "" ] +then + exit 1 +fi + diff --git a/tools/bazelify_tests/test/BUILD b/tools/bazelify_tests/test/BUILD new file mode 100644 index 00000000000..5aedb1b20e6 --- /dev/null +++ b/tools/bazelify_tests/test/BUILD @@ -0,0 +1,158 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("//bazel:grpc_build_system.bzl", "grpc_package") +load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") +load(":portability_tests.bzl", "generate_run_tests_portability_tests") + +licenses(["notice"]) + +grpc_package(name = "tools/bazelify_tests/test") + +generate_run_tests_portability_tests(name = "portability_tests_linux") + +# C/C++ +grpc_run_tests_py_test( + name = "runtests_c_linux_dbg", + size = "enormous", + args = [ + "-l c -c dbg", + ], + docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", +) + +grpc_run_tests_py_test( + name = "runtests_c_linux_opt", + size = "enormous", + args = [ + "-l c -c opt", + ], + docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", +) + +grpc_run_tests_py_test( + name = "runtests_cpp_linux_dbg_build_only", + size = "enormous", + args = [ + "-l c++ -c dbg --build_only", + ], + docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", +) + +grpc_run_tests_py_test( + name = "runtests_cpp_linux_opt_build_only", + size = "enormous", + args = [ + "-l c++ -c opt --build_only", + ], + docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", +) + +# TODO(jtattermusch): Reintroduce ruby tests once they pass. +# # Ruby +# grpc_run_tests_py_test( +# name = "runtests_ruby_linux_dbg", +# size = "enormous", +# args = [ +# "-l ruby -c dbg", +# ], +# docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version", +# prepare_script = ":prepare_ruby.sh", +# use_login_shell = True, # ruby's docker image uses RVM which wierdly requires login shell +# ) + +# grpc_run_tests_py_test( +# name = "runtests_ruby_linux_opt", +# size = "enormous", +# args = [ +# "-l ruby -c opt", +# ], +# docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version", +# prepare_script = ":prepare_ruby.sh", +# use_login_shell = True, # ruby's docker image uses RVM which wierdly requires login shell +# ) + +# PHP +grpc_run_tests_py_test( + name = "runtests_php_linux_dbg", + size = "enormous", + args = [ + "-l php7 -c dbg", + ], + docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version", +) + +grpc_run_tests_py_test( + name = "runtests_php_linux_opt", + size = "enormous", + args = [ + "-l php7 -c opt", + ], + docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version", +) + +# TODO(jtattermusch): Reintroduce python tests once they pass. +# # Python +# grpc_run_tests_py_test( +# name = "runtests_python_linux_opt", +# size = "enormous", +# args = [ +# "-l python -c opt", +# ], +# docker_image_version = "tools/dockerfile/test/python_debian11_default_x64.current_version", +# ) + +# C# +grpc_run_tests_py_test( + name = "runtests_csharp_linux_dbg", + size = "enormous", + args = [ + "-l csharp -c dbg", + ], + docker_image_version = "tools/dockerfile/test/csharp_debian11_x64.current_version", +) + +grpc_run_tests_py_test( + name = "runtests_csharp_linux_opt", + size = "enormous", + args = [ + "-l csharp -c opt", + ], + docker_image_version = "tools/dockerfile/test/csharp_debian11_x64.current_version", +) + +test_suite( + name = "basic_tests_linux", + tests = [ + ":runtests_c_linux_dbg", + ":runtests_c_linux_opt", + ":runtests_cpp_linux_dbg_build_only", + ":runtests_cpp_linux_opt_build_only", + ":runtests_csharp_linux_dbg", + ":runtests_csharp_linux_opt", + ":runtests_php_linux_dbg", + ":runtests_php_linux_opt", + #":runtests_python_linux_opt", + #":runtests_ruby_linux_dbg", + #":runtests_ruby_linux_opt", + ], +) + +test_suite( + name = "all_tests_linux", + tests = [ + ":basic_tests_linux", + ":portability_tests_linux", + ], +) diff --git a/tools/bazelify_tests/test/portability_tests.bzl b/tools/bazelify_tests/test/portability_tests.bzl new file mode 100644 index 00000000000..21cc04c2ee0 --- /dev/null +++ b/tools/bazelify_tests/test/portability_tests.bzl @@ -0,0 +1,91 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Generates portability tests. +""" + +load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") + +def _safe_language_name(name): + """Character '+' isn't allowed in bazel target name""" + return name.replace("+", "p") + +def generate_run_tests_portability_tests(name): + """Generates run_tests_py portability test targets. + + Args: + name: Name of the test suite that will be generated. + """ + test_names = [] + + # portability C x86 + grpc_run_tests_py_test( + name = "runtests_c_linux_dbg_x86", + args = ["-l c -c dbg"], + docker_image_version = "tools/dockerfile/test/cxx_debian11_x86.current_version", + size = "enormous", + ) + test_names.append("runtests_c_linux_dbg_x86") + + # C and C++ with no-exceptions on Linux + for language in ["c", "c++"]: + test_name = "runtests_%s_linux_dbg_noexcept_build_only" % _safe_language_name(language) + grpc_run_tests_py_test( + name = test_name, + args = ["-l %s --config noexcept --build_only" % language], + docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", + size = "enormous", + ) + test_names.append(test_name) + + # C and C++ under different compilers + for language in ["c", "c++"]: + compiler_configs = [ + # TODO(b/283304471): Add 'gcc10.2_openssl102' once possible + ["gcc_7", "", "tools/dockerfile/test/cxx_gcc_7_x64.current_version"], + ["gcc_12", "--cmake_configure_extra_args=-DCMAKE_CXX_STANDARD=20", "tools/dockerfile/test/cxx_gcc_12_x64.current_version"], + # TODO(jtattermusch): Re-enable once the build can finish in reasonable time (looks like ccache is not being used?) + #["gcc_musl", "", "tools/dockerfile/test/cxx_alpine_x64.current_version"], + ["clang_6", "--cmake_configure_extra_args=-DCMAKE_C_COMPILER=clang --cmake_configure_extra_args=-DCMAKE_CXX_COMPILER=clang++", "tools/dockerfile/test/cxx_clang_6_x64.current_version"], + ["clang_15", "--cmake_configure_extra_args=-DCMAKE_C_COMPILER=clang --cmake_configure_extra_args=-DCMAKE_CXX_COMPILER=clang++", "tools/dockerfile/test/cxx_clang_15_x64.current_version"], + ] + + for compiler_name, args, docker_image_version in compiler_configs: + test_name = "runtests_%s_linux_dbg_%s_build_only" % (_safe_language_name(language), compiler_name) + grpc_run_tests_py_test( + name = test_name, + args = ["-l %s -c dbg %s --build_only" % (language, args)], + docker_image_version = docker_image_version, + size = "enormous", + ) + test_names.append(test_name) + + # TODO(jtattermusch): Reintroduce the test once it passes. + # Python on alpine + #grpc_run_tests_py_test( + # name = "runtests_python_linux_dbg_alpine", + # args = [ + # "-l python -c dbg --compiler python_alpine", + # ], + # docker_image_version = "tools/dockerfile/test/python_alpine_x64.current_version", + # size = "enormous", + #) + #test_names.append("runtests_python_linux_dbg_alpine") + + # Generate test suite that allows easily running all portability tests. + native.test_suite( + name = name, + tests = [(":%s" % test_name) for test_name in test_names], + ) diff --git a/tools/bazelify_tests/workspace_status_cmd.sh b/tools/bazelify_tests/workspace_status_cmd.sh new file mode 100755 index 00000000000..83bafd2140a --- /dev/null +++ b/tools/bazelify_tests/workspace_status_cmd.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +GRPC_GIT_COMMIT="$(git rev-parse HEAD)" +echo "STABLE_GRPC_GIT_COMMIT ${GRPC_GIT_COMMIT}" + +# produce a value that has name and commit for every submodule +echo -n 'STABLE_GRPC_SUBMODULE_ARCHIVES' +git submodule --quiet foreach 'git_commit="$(git rev-parse HEAD)"; echo -n " ${name}/${git_commit}"'; echo "" + +# set info about whether the git workspace is clean/dirty and checksum of git patch +if [ "$(git status --porcelain)" == "" ] +then + echo "STABLE_GRPC_GIT_WORKSPACE_DIRTY false" +else + echo "STABLE_GRPC_GIT_WORKSPACE_DIRTY true" + echo "STABLE_GRPC_UNCOMMITED_PATCH_CHECKSUM $(git diff HEAD | sha256sum | cut -f1 -d' ')" +fi + +# Since only --workspace_status_command is allowed by bazel, also include +# status from the "default" workspace status command. +tools/remote_build/workspace_status_kokoro.sh diff --git a/tools/dockerfile/push_testing_images.sh b/tools/dockerfile/push_testing_images.sh index f922a59fcc0..8c136b46ba0 100755 --- a/tools/dockerfile/push_testing_images.sh +++ b/tools/dockerfile/push_testing_images.sh @@ -208,6 +208,15 @@ do fi done +if [ "${CHECK_MODE}" != "" ] +then + # Check that dockerimage_current_versions.bzl is up to date. + CHECK_MODE="${CHECK_MODE}" tools/bazelify_tests/generate_dockerimage_current_versions_bzl.sh || CHECK_FAILED=true +else + # Regenerate dockerimage_current_versions.bzl + tools/bazelify_tests/generate_dockerimage_current_versions_bzl.sh +fi + if [ "${CHECK_FAILED}" != "" ] then echo "ERROR: Some checks have failed." diff --git a/tools/internal_ci/linux/grpc_bazel_rbe_nonbazel.cfg b/tools/internal_ci/linux/grpc_bazel_rbe_nonbazel.cfg new file mode 100644 index 00000000000..8bafa25acac --- /dev/null +++ b/tools/internal_ci/linux/grpc_bazel_rbe_nonbazel.cfg @@ -0,0 +1,52 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_bazel_rbe.sh" +timeout_mins: 90 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/resultstore_api_key" + +bazel_setting { + # In order for Kokoro to recognize this as a bazel build and publish the bazel resultstore link, + # the bazel_setting section needs to be present and "upsalite_frontend_address" needs to be + # set. The rest of configuration from bazel_setting is unused (we configure everything when bazel + # command is invoked). + upsalite_frontend_address: "https://source.cloud.google.com" +} + +env_vars { + # flags will be passed to bazel invocation + key: "BAZEL_FLAGS" + value: "--genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh --cache_test_results=no" +} + +env_vars { + # tests to run by the bazel invocation + key: "BAZEL_TESTS" + value: "//tools/bazelify_tests/test:all_tests_linux" +} + +env_vars { + key: "UPLOAD_TEST_RESULTS" + value: "true" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_bazel_rbe_nonbazel.cfg b/tools/internal_ci/linux/pull_request/grpc_bazel_rbe_nonbazel.cfg new file mode 100644 index 00000000000..e4eac5a4bff --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_bazel_rbe_nonbazel.cfg @@ -0,0 +1,47 @@ +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_bazel_rbe.sh" +timeout_mins: 90 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/resultstore_api_key" + +bazel_setting { + # In order for Kokoro to recognize this as a bazel build and publish the bazel resultstore link, + # the bazel_setting section needs to be present and "upsalite_frontend_address" needs to be + # set. The rest of configuration from bazel_setting is unused (we configure everything when bazel + # command is invoked). + upsalite_frontend_address: "https://source.cloud.google.com" +} + +env_vars { + # flags will be passed to bazel invocation + key: "BAZEL_FLAGS" + value: "--genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh" +} + +env_vars { + # tests to run by the bazel invocation + key: "BAZEL_TESTS" + value: "//tools/bazelify_tests/test:all_tests_linux" +} diff --git a/tools/remote_build/include/rbe_base_config.bazelrc b/tools/remote_build/include/rbe_base_config.bazelrc index a7fc8c152b5..0f98d28e32a 100644 --- a/tools/remote_build/include/rbe_base_config.bazelrc +++ b/tools/remote_build/include/rbe_base_config.bazelrc @@ -41,3 +41,6 @@ build --bes_backend=grpcs://buildeventservice.googleapis.com build --bes_timeout=600s build --bes_results_url="https://source.cloud.google.com/results/invocations/" build --bes_instance_name=grpc-testing + +# Avoid timeouts for actions that don't write output for long time (See b/143346671 and b/143134296) +build --grpc_keepalive_time=10m diff --git a/tools/remote_build/linux_docker_sandbox.bazelrc b/tools/remote_build/linux_docker_sandbox.bazelrc new file mode 100644 index 00000000000..cb27ec26d92 --- /dev/null +++ b/tools/remote_build/linux_docker_sandbox.bazelrc @@ -0,0 +1,45 @@ +#@IgnoreInspection BashAddShebang +# Copyright 2023 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# bazelrc file for running gRPC tests in a docker sandbox to help +# with debugging issues with RBE. +# TODO(jtattermusch): Settings in this file are currently EXPERIMENTAL. Use +# at your own risk. + +# TODO(jtattermusch): Consider including some configuration settings from +# tools/remote_build/include/rbe_remote_execution.bazelrc + +build --experimental_docker_verbose +build --experimental_enable_docker_sandbox + +build --spawn_strategy=docker +build --strategy=Javac=docker +build --strategy=Closure=docker +build --genrule_strategy=docker + +# Next section is linux-specific RBE configuration +build --crosstool_top=//third_party/toolchains:rbe_linux_default_toolchain_suite +build --extra_toolchains=//third_party/toolchains:rbe_linux_default_cc_toolchain +# Use custom execution platforms defined in third_party/toolchains +build --extra_execution_platforms=//third_party/toolchains:rbe_linux_default_platform +build --host_platform=//third_party/toolchains:rbe_linux_default_platform +build --platforms=//third_party/toolchains:rbe_linux_default_platform + +# we assume the default bazel RBE build is on linux, +# so filter out stuff that should not be built or run there. +build --test_tag_filters=-no_linux +build --build_tag_filters=-no_linux + +import %workspace%/tools/remote_build/include/test_config_common.bazelrc diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index e83f5082f1f..70ba6204206 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -305,7 +305,9 @@ class CLanguage(object): print("should never reach here.") sys.exit(1) - self._cmake_configure_extra_args = [] + self._cmake_configure_extra_args = list( + self.args.cmake_configure_extra_args + ) self._cmake_generator_windows = cmake_generator # required to pass as cmake "-A" configuration for VS builds (but not for Ninja) self._cmake_architecture_windows = ( @@ -329,7 +331,11 @@ class CLanguage(object): ( self._docker_distro, self._cmake_configure_extra_args, - ) = self._compiler_options(self.args.use_docker, self.args.compiler) + ) = self._compiler_options( + self.args.use_docker, + self.args.compiler, + self.args.cmake_configure_extra_args, + ) if self.args.arch == "x86": # disable boringssl asm optimizations when on x86 @@ -544,8 +550,14 @@ class CLanguage(object): "-DCMAKE_CXX_COMPILER=clang++%s" % version_suffix, ] - def _compiler_options(self, use_docker, compiler): + def _compiler_options( + self, use_docker, compiler, cmake_configure_extra_args + ): """Returns docker distro and cmake configure args to use for given compiler.""" + if cmake_configure_extra_args: + # only allow specifying extra cmake args for "vanilla" compiler + _check_compiler(compiler, ["default", "cmake"]) + return ("nonexistent_docker_distro", cmake_configure_extra_args) if not use_docker and not _is_use_docker_child(): # if not running under docker, we cannot ensure the right compiler version will be used, # so we only allow the non-specific choices. @@ -1824,6 +1836,12 @@ argp.add_argument( nargs="?", help="Upload test results to a specified BQ table.", ) +argp.add_argument( + "--cmake_configure_extra_args", + default=[], + nargs="+", + help="Extra arguments that will be passed to the cmake configure command. Only works for C/C++.", +) args = argp.parse_args() flaky_tests = set()