[bazel] Teach bazel how to run non-bazel test suites under docker locally and on RBE (#33707)
Add "bazelified" non-bazel tests. See tools/bazelify_tests/README.md for the core idea. - add a bunch of test targets that run under docker and execute tests that correspond to `run_tests.py -l LANG ...` - many more tests can be added in the future - to enable running some of the C/C++ portability tests easily, added support for `--cmake_extra_configure_args` in run_tests.py (the change is fairly small). Example passing build that shows how test results are structured: https://source.cloud.google.com/results/invocations/21295351-a3e3-4be1-b6e9-aaf52195a044/targetspull/33960/head
Jan Tattermusch
2 years ago
committed by
GitHub
parent
0747da86c7
commit
2b0a9d3916
17 changed files with 1038 additions and 3 deletions
@ -0,0 +1,51 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
load("//bazel:grpc_build_system.bzl", "grpc_package") |
||||
|
||||
licenses(["notice"]) |
||||
|
||||
# TODO(jtattermusch): adjust the visibility |
||||
grpc_package( |
||||
name = "tools/bazelify_tests", |
||||
visibility = "public", |
||||
) |
||||
|
||||
exports_files([ |
||||
"grpc_run_tests_py_test.sh", |
||||
"grpc_build_artifact_task.sh", |
||||
]) |
||||
|
||||
genrule( |
||||
name = "grpc_repo_archive", |
||||
srcs = |
||||
[ |
||||
"grpc_repo_archive.sh", |
||||
], |
||||
outs = [ |
||||
"grpc_repo_archive.tar.gz", |
||||
"grpc_repo_archive_with_submodules.tar.gz", |
||||
], |
||||
cmd = "$(location grpc_repo_archive.sh) $(location grpc_repo_archive.tar.gz) $(location grpc_repo_archive_with_submodules.tar.gz)", |
||||
local = True, |
||||
stamp = 1, |
||||
tags = [ |
||||
"manual", |
||||
], |
||||
) |
||||
|
||||
alias( |
||||
name = "grpc_repo_archive_with_submodules", |
||||
actual = ":grpc_repo_archive", |
||||
) |
||||
@ -0,0 +1,45 @@ |
||||
# Non-Bazel native tests |
||||
|
||||
This directory contains logic that wraps builds and tests from the non-bazel realm to make them runnable under bazel. |
||||
|
||||
Examples: cmake builds, run_tests.py tests, artifacts, distribtests etc. |
||||
|
||||
NOTE: all tests and their setup under this directory are currently EXPERIMENTAL. |
||||
|
||||
## How it works |
||||
|
||||
The `//tools/bazelify_tests:repo_archive` target produces an archive that contains grpc at the current head with all its submodules. The rule uses a few tricks to achieve this: |
||||
|
||||
* Uses a workspace status command to obtain the commit SHAs of grpc and all submodules from the workspace. |
||||
* When running, it actually jailbreaks from the bazel execroot to access the bazel workspace and create the necessary archives. |
||||
* The produced archives are deterministic (they have the same checksum if neither grpc or its submodules have changed). |
||||
* The target is defined in such a way so that it behaves "reasonably" from bazel's perspective (always re-runs if commit SHAs have changed, can be cached if not). |
||||
|
||||
After grpc source code is archived, the "bazelified" tests basically depend on the archive and they run a script under a docker container. |
||||
The script unpacks the archived grpc code and creates a temporary workspace (under bazel's target execroot) and then performs the actions that are needed |
||||
(e.g. run the run_tests.py test harness, run cmake build etc). |
||||
|
||||
There are two ways the test targets can run under a docker container: |
||||
|
||||
* When running on RBE, all actions run under a docker container by definition. |
||||
* When running locally, bazel will start a docker container for each action when [docker sandbox](https://bazel.build/remote/sandbox) is used. |
||||
(Note that the docker sandbox currently [doesn't work on windows](https://github.com/bazelbuild/bazel/issues/19101)) |
||||
|
||||
In both cases, the docker image which is used for any given action is determined by the action's `exec_properties` and can be specified as a default |
||||
(e.g. by RBE toolchain or by setting `--experimental_docker_image` flag) or explicitly for each action. For most tests in this directory, |
||||
the test rules actually configure the `exec_properties` for you, based on selecting one of the gRPC's testing docker images. |
||||
|
||||
## Run tests on RBE |
||||
|
||||
``` |
||||
# "--genrule_strategy=remote,local" allows fallback to local execution if action doesn't support running remotely |
||||
# (required to be able to run the //tools/bazelify_tests:repo_archive target). |
||||
tools/bazel --bazelrc=tools/remote_build/linux.bazelrc test --genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh //tools/bazelify_tests/test:basic_tests_linux |
||||
``` |
||||
|
||||
## Run tests locally under bazel's docker sandbox |
||||
|
||||
``` |
||||
tools/bazel --bazelrc=tools/remote_build/linux_docker_sandbox.bazelrc test --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh //tools/bazelify_tests/test:basic_tests_linux |
||||
``` |
||||
|
||||
@ -0,0 +1,103 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
""" |
||||
Contains macros used for running bazelified tests. |
||||
""" |
||||
|
||||
load(":dockerimage_current_versions.bzl", "DOCKERIMAGE_CURRENT_VERSIONS") |
||||
|
||||
def grpc_run_tests_py_test(name, args = [], data = [], size = "medium", timeout = None, tags = [], exec_compatible_with = [], flaky = None, docker_image_version = None, use_login_shell = None, prepare_script = None): |
||||
"""Execute an run_tests.py-harness style test under bazel. |
||||
|
||||
Args: |
||||
name: The name of the test. |
||||
args: The args to supply to the test binary. |
||||
data: Data dependencies. |
||||
size: The size of the test. |
||||
timeout: The test timeout. |
||||
tags: The tags for the test. |
||||
exec_compatible_with: A list of constraint values that must be |
||||
satisifed for the platform. |
||||
flaky: Whether this test is flaky. |
||||
docker_image_version: The docker .current_version file to use for docker containerization. |
||||
use_login_shell: If True, the run_tests.py command will run under a login shell. |
||||
prepare_script: Optional script that will be sourced before run_tests.py runs. |
||||
""" |
||||
|
||||
data = [ |
||||
"//tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz", |
||||
] + data |
||||
|
||||
args = [ |
||||
"$(location //tools/bazelify_tests:grpc_repo_archive_with_submodules.tar.gz)", |
||||
] + args |
||||
|
||||
srcs = [ |
||||
"//tools/bazelify_tests:grpc_run_tests_py_test.sh", |
||||
] |
||||
|
||||
env = {} |
||||
|
||||
if use_login_shell: |
||||
env["GRPC_RUNTESTS_USE_LOGIN_SHELL"] = "1" |
||||
|
||||
if prepare_script: |
||||
data = data + [prepare_script] |
||||
env["GRPC_RUNTESTS_PREPARE_SCRIPT"] = "$(location " + prepare_script + ")" |
||||
|
||||
# Enable ccache by default. This is important for speeding up the C++ cmake build, |
||||
# which isn't very efficient and tends to recompile some source files multiple times. |
||||
# Even though only the local disk cache is enabled (local to the docker container, |
||||
# so will be thrown away after the bazel actions finishes), ccache still speeds up |
||||
# the C++ build significantly. |
||||
# TODO(jtattermusch): find a cleaner way to toggle ccache for builds. |
||||
env["GRPC_BUILD_ENABLE_CCACHE"] = "true" |
||||
|
||||
# TODO(jtattermusch): use rbe_exec_properties helpers instead of manually specifying |
||||
# the properties, which is fragile. |
||||
exec_properties = { |
||||
"dockerNetwork": "standard", # TODO(jtattermusch): look into deactivating network for some actions |
||||
"label:workload": "misc", # always use a dedicated "misc" pool for running bazelified tests |
||||
"label:machine_size": "misc_large", # needed to override the default value of "small". |
||||
} |
||||
if docker_image_version: |
||||
image_spec = DOCKERIMAGE_CURRENT_VERSIONS.get(docker_image_version, None) |
||||
if not image_spec: |
||||
fail("Version info for docker image '%s' not found in dockerimage_current_versions.bzl" % docker_image_version) |
||||
exec_properties["container-image"] = image_spec |
||||
|
||||
# since the tests require special bazel args, only run them when explicitly requested |
||||
tags = ["manual"] + tags |
||||
|
||||
# TODO(jtattermusch): find a way to ensure that action can only run under docker sandbox or remotely |
||||
# to avoid running it outside of a docker container by accident. |
||||
|
||||
test_args = { |
||||
"name": name, |
||||
"srcs": srcs, |
||||
"tags": tags, |
||||
"args": args, |
||||
"flaky": flaky, |
||||
"data": data, |
||||
"size": size, |
||||
"env": env, |
||||
"timeout": timeout, |
||||
"exec_compatible_with": exec_compatible_with, |
||||
"exec_properties": exec_properties, |
||||
} |
||||
|
||||
native.sh_test( |
||||
**test_args |
||||
) |
||||
@ -0,0 +1,117 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
""" |
||||
This file is generated by generate_dockerimage_current_versions_bzl.sh |
||||
It makes the info from testing docker image *.current_version files |
||||
accessible to bazel builds. |
||||
""" |
||||
|
||||
DOCKERIMAGE_CURRENT_VERSIONS = { |
||||
"third_party/rake-compiler-dock/rake_aarch64-linux.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_aarch64-linux@sha256:61a46ab67972990aea77024817d29ca6fa43d2639fe4aaef9c30e031f84519a9", |
||||
"third_party/rake-compiler-dock/rake_arm64-darwin.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_arm64-darwin@sha256:e0eb1f9f632fb18d4f244b7297d1a5e7cf60ae58e649ac5b2f8ac6266ea07128", |
||||
"third_party/rake-compiler-dock/rake_x64-mingw-ucrt.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw-ucrt@sha256:63490b0000c6011a19983fef637efc69a2ae0f67b7a4e29cd36db53c881e908d", |
||||
"third_party/rake-compiler-dock/rake_x64-mingw32.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x64-mingw32@sha256:63ece6e9b336b7cbf66eaa0201505b0579ac06cd7802f19b44c3a816d5617c17", |
||||
"third_party/rake-compiler-dock/rake_x86-linux.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-linux@sha256:71e3afca0843bf7bd5da7fa04bff40ad976e76aa5867936166b30251d0a692d8", |
||||
"third_party/rake-compiler-dock/rake_x86-mingw32.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86-mingw32@sha256:629be8f57e2d50b123584f2cfa00ff5b968cc2cc3b2a6b874acd07100a3eb96d", |
||||
"third_party/rake-compiler-dock/rake_x86_64-darwin.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-darwin@sha256:8dd11cad778d9fc01c3555a57254016f5db7227309d24f50a192a6db80d4a51c", |
||||
"third_party/rake-compiler-dock/rake_x86_64-linux.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rake_x86_64-linux@sha256:9aa77587fa4d4c25c91d0ccca8eb806cf0738a4b67eb4b54d40324185658194e", |
||||
"tools/dockerfile/distribtest/cpp_debian10_aarch64_cross_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cpp_debian10_aarch64_cross_x64@sha256:15eeafcd816cb32a0d44da22f654749352a92fec9626dc028b39948897d5bea3", |
||||
"tools/dockerfile/distribtest/cpp_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cpp_debian10_x64@sha256:904e3db8521697768f94aa08230063b474246184e126f74a41b98a6f4aaf6a49", |
||||
"tools/dockerfile/distribtest/csharp_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_alpine_x64@sha256:d018105349fcabdc3aa0649c1381d840c613df6b442a53a751d7dc839a80d429", |
||||
"tools/dockerfile/distribtest/csharp_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_centos7_x64@sha256:ec715dd5fbd621789e7598c8d4ac346a7b4037b0cc83fbb29990dc8e4c1f1a13", |
||||
"tools/dockerfile/distribtest/csharp_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian10_x64@sha256:8c3838e731da70566adc6f989f2c29351fdb2f629e8797928699fff24b3a0938", |
||||
"tools/dockerfile/distribtest/csharp_dotnet31_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet31_x64@sha256:fee52df6064ff84bc9af644c2ea17ab579de3401e3a167d0d43383c24f0d500f", |
||||
"tools/dockerfile/distribtest/csharp_dotnet5_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_dotnet5_x64@sha256:408425cd74bb8b79a3b09a64ea6c54f6cdc0e757a3469f31effc017a7187e442", |
||||
"tools/dockerfile/distribtest/csharp_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_ubuntu1604_x64@sha256:e0f44406df14a28ce0a0f4e26c74c95f0fa5dddadf1fdbb2a3793b7c8ef8fa63", |
||||
"tools/dockerfile/distribtest/php7_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian10_x64@sha256:e760a60f2dce2dada571d9b07447a9f99ffeeb366a309dbbb5dc0a43991c22dc", |
||||
"tools/dockerfile/distribtest/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:699ac7b86199406fa27e88f30a1c623ef34ac33f6d9330fd13a6f6457ee4e19f", |
||||
"tools/dockerfile/distribtest/python_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_arch_x64@sha256:2c1adadeb010e107132cf5137f32a2d18727796631245b110cc74f69c07502e1", |
||||
"tools/dockerfile/distribtest/python_buster_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_buster_x64@sha256:e501dc8e2f4ab9cd4382974759a879a27c065c8fed5327f538764298fc5c4972", |
||||
"tools/dockerfile/distribtest/python_buster_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_buster_x86@sha256:185fbb174525d67b6146f4d233c804c589b0b57d783bb1bf95bc47cfe792754e", |
||||
"tools/dockerfile/distribtest/python_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_centos7_x64@sha256:39afaa687cb8516eef1621ed789326fdde2014fd3c81d11a1ded72f2d5285fe1", |
||||
"tools/dockerfile/distribtest/python_dev_alpine3.7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_alpine3.7_x64@sha256:7c08f67211a49eb72ad08c29de5c64a914c066d9c1670b712e717571b8d5c7e2", |
||||
"tools/dockerfile/distribtest/python_dev_arch_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_arch_x64@sha256:29f179ef2083ee6addd57e90f58781fdc1cb5dc3dd3e228da1af38785b921f35", |
||||
"tools/dockerfile/distribtest/python_dev_buster_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_buster_x64@sha256:e30d6efdeac24e5136cc169d503a239df22147bfb121d27feb1f87d58a8fe64e", |
||||
"tools/dockerfile/distribtest/python_dev_buster_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_buster_x86@sha256:179146fd5d5cc15846c6bf0284c2e261f383caf944559d2d9f7a5af0e0f7152d", |
||||
"tools/dockerfile/distribtest/python_dev_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_centos7_x64@sha256:e6e9a1b23a0a543050db91e17d621aa899bad04308adaf961c11fa88ba941a95", |
||||
"tools/dockerfile/distribtest/python_dev_fedora34_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_fedora34_x64@sha256:20bc3a6283a99407eb637b3cde1ff3e1288a1e21388a1dc385c2b4df5a1eb1c2", |
||||
"tools/dockerfile/distribtest/python_dev_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_ubuntu1604_x64@sha256:167134c3a43e7d2608c893cc98a5066eae93c6af97ef5a1e69d643cbc7fefc43", |
||||
"tools/dockerfile/distribtest/python_dev_ubuntu1804_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_ubuntu1804_x64@sha256:157a89cd6d0e69b89ac1975e0314aade556a35aafbaa5fe9f9890f90321d6c89", |
||||
"tools/dockerfile/distribtest/python_dev_ubuntu2004_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_dev_ubuntu2004_x64@sha256:91f0d88c43ec52ecd63f99acb424c88ff9a67fa046fae207a75e99bee37eef11", |
||||
"tools/dockerfile/distribtest/python_fedora34_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_fedora34_x64@sha256:5aa8e41d627ddd6bc10aae6b12d25ded90ba8554a63b279f43f44e0d6cf001dd", |
||||
"tools/dockerfile/distribtest/python_opensuse_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_opensuse_x64@sha256:da52566b078d10e537aa219e59641731a57e5dc7d17d6737f5e5a7d447acf5cc", |
||||
"tools/dockerfile/distribtest/python_python38_buster_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_python38_buster_aarch64@sha256:487b9af2ad1459ee2630694e61074d4ac525d4f90b2bdb026dbf6f77fb3e9878", |
||||
"tools/dockerfile/distribtest/python_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu1604_x64@sha256:44a821a9f5431122c2e239ba35cf181c2fde84a5d866e8add338599565881492", |
||||
"tools/dockerfile/distribtest/python_ubuntu1804_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu1804_x64@sha256:edcd5f342d77ad9129cc0a0e6988b47b144815e7a93091d5b45e850111eefbcf", |
||||
"tools/dockerfile/distribtest/python_ubuntu2004_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_ubuntu2004_x64@sha256:342e9dc23b674ad256b220745745be818708a1baa25a2690f0d4f777e28a22a3", |
||||
"tools/dockerfile/distribtest/ruby_centos7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_centos7_x64@sha256:4d529b984b78ca179086f7f9b416605e2d9a96ca0a28a71f4421bb5ffdc18f96", |
||||
"tools/dockerfile/distribtest/ruby_debian10_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64@sha256:1298c39c950b2a48261555b6cff1ae66230a5020f100d3b381759285f0caf84e", |
||||
"tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_6.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_2_6@sha256:3ef9a84a77276f2ccaca3c97336573f76ed02719645e4bb67ccdf1f33da99fc8", |
||||
"tools/dockerfile/distribtest/ruby_debian10_x64_ruby_2_7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_2_7@sha256:5ee26ad3abe2683c9a8ee03987ab0ae63f50793c3d3f5e4be6e6cbacb4556fcf", |
||||
"tools/dockerfile/distribtest/ruby_debian10_x64_ruby_3_0.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian10_x64_ruby_3_0@sha256:9190da90a2a95eca1370cef64dcba7ddee9f59cc7487093da6711c1280a0b0f9", |
||||
"tools/dockerfile/distribtest/ruby_ubuntu1604_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_ubuntu1604_x64@sha256:e0276968184a6c1e16de4e6afbbd469df91b27e40d061340841c76e864fdcb50", |
||||
"tools/dockerfile/distribtest/ruby_ubuntu1804_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_ubuntu1804_x64@sha256:d38b3dd34cffc027e9a1bf82bc7ace75b8a9829c2d04d5cf7cc8323655edd27a", |
||||
"tools/dockerfile/grpc_artifact_centos6_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x64@sha256:3285047265ea2b7c5d4df4c769b2d05f56288d947c75e16d27ae2dee693f791b", |
||||
"tools/dockerfile/grpc_artifact_centos6_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_centos6_x86@sha256:19783239da92208f0f39cf563529cd02e889920497ef81c60d20391fa998af62", |
||||
"tools/dockerfile/grpc_artifact_protoc_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_protoc_aarch64@sha256:1a3957f32e81259e6f3c602bd67feb132ebc5a5f23e9cb0bf63ba34b91185982", |
||||
"tools/dockerfile/grpc_artifact_python_linux_armv7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_linux_armv7@sha256:4f817dece74bbdc7c4fccdc9b0a25cefb9101781a60bf0bb827e533e79f9b1f2", |
||||
"tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_aarch64@sha256:d56ea4394ea5ea9d09f940d1dba31e6196a8e919f60c6a4966a9192faa997f11", |
||||
"tools/dockerfile/grpc_artifact_python_manylinux2014_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x64@sha256:67ab746e6da576606ebf41ad81027ad897544445fb93d5d5ca5f9d9b5428ec84", |
||||
"tools/dockerfile/grpc_artifact_python_manylinux2014_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_manylinux2014_x86@sha256:993a963ac3985f8634951e1573d34e24b3868dfff3ad4ae4875dd2c47b73224f", |
||||
"tools/dockerfile/grpc_artifact_python_musllinux_1_1_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x64@sha256:09bf18cc793d55cfc48d8e88b8b6e6914e9df2b35ec417fe77a4e20bfa251df7", |
||||
"tools/dockerfile/grpc_artifact_python_musllinux_1_1_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_artifact_python_musllinux_1_1_x86@sha256:0512449e7d218c7687eb447701c8c6a33153a722722b76b2423ec58440a027de", |
||||
"tools/dockerfile/interoptest/grpc_interop_aspnetcore.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_aspnetcore@sha256:8e2e732e78724a8382c340dca72e7653c5f82c251a3110fa2874cc00ba538878", |
||||
"tools/dockerfile/interoptest/grpc_interop_cxx.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_cxx@sha256:a69a1ed729137c3ea347f0a3488524573285be7832dd74cec830db57b61a9b8c", |
||||
"tools/dockerfile/interoptest/grpc_interop_dart.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_dart@sha256:5e335005b27709f0882c5723affafa55094bd27a0cda7ce91c718deed157f2bb", |
||||
"tools/dockerfile/interoptest/grpc_interop_go.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go@sha256:889e7ff34399a5e16af87940d1eaa239e56da307f7faca3f8f1d28379c2e3df3", |
||||
"tools/dockerfile/interoptest/grpc_interop_go1.11.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.11@sha256:29cde59287843a3208c0cabeaf430cf813846a738c8a1b9692e68b54bbbdcc2d", |
||||
"tools/dockerfile/interoptest/grpc_interop_go1.16.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.16@sha256:d5b2b0c02e7a8196fea704196a8221994983c22eece2ac2324e095e8972a957f", |
||||
"tools/dockerfile/interoptest/grpc_interop_go1.19.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.19@sha256:889e7ff34399a5e16af87940d1eaa239e56da307f7faca3f8f1d28379c2e3df3", |
||||
"tools/dockerfile/interoptest/grpc_interop_go1.8.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_go1.8@sha256:7830a301b37539252c592b9cd7fa30a6142d0afc717a05fc8d2b82c74fb45efe", |
||||
"tools/dockerfile/interoptest/grpc_interop_http2.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_http2@sha256:e3f247d8038374848fadf7215b841e3575c0b2a4217feb503a79b8004b164c5a", |
||||
"tools/dockerfile/interoptest/grpc_interop_java.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_java@sha256:d9210764071662ba2f377dafcaff4b743f41e4dff1876dd47df7b1c6950f88af", |
||||
"tools/dockerfile/interoptest/grpc_interop_node.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_node@sha256:337e9995563e4f569b4daf843d0a2af0619e086481ce3ba3f888434eb2ddc28b", |
||||
"tools/dockerfile/interoptest/grpc_interop_nodepurejs.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_nodepurejs@sha256:9061077a17eb6f2306af563ed85c12630480f6d6ce15919d67ef5567dbab559e", |
||||
"tools/dockerfile/interoptest/grpc_interop_php7.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_php7@sha256:09f4b895117c81506c423360b617917d06d3f7f0b78e4cca25eaec547ba6991e", |
||||
"tools/dockerfile/interoptest/grpc_interop_python.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_python@sha256:fef1247f8256be2b9841331e7d21b0046da21a4a6d34a62addb36f62124725cf", |
||||
"tools/dockerfile/interoptest/grpc_interop_pythonasyncio.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_pythonasyncio@sha256:bd4cdc8a71ef339193e178ce20d2b47a0b2aa25fc86c0b5740b9d86a2d4a0caa", |
||||
"tools/dockerfile/interoptest/grpc_interop_ruby.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/grpc_interop_ruby@sha256:0de52450b29cf91365e623b020cd97722c307510ba1813bee09264e0a49acdbc", |
||||
"tools/dockerfile/interoptest/lb_interop_fake_servers.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/lb_interop_fake_servers@sha256:b89a51dd9147e1293f50ee64dd719fce5929ca7894d3770a3d80dbdecb99fd52", |
||||
"tools/dockerfile/test/android_ndk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/android_ndk@sha256:2bddf36ae504968b35f97e4a6c9b74864473689e84049675c30afb70f868d897", |
||||
"tools/dockerfile/test/bazel.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel@sha256:1118150d9d9479787165fff49f660a3dc633f1c57604305460172fc1916aa022", |
||||
"tools/dockerfile/test/bazel_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/bazel_arm64@sha256:dd4a71e2852faf24b45e21445331f400bb278c83f6ad6d7d89d43c9ac564d529", |
||||
"tools/dockerfile/test/binder_transport_apk.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/binder_transport_apk@sha256:85341f035e44601a93d16ff5b9b5810a0da313af03e2a76cf4135144633e0bab", |
||||
"tools/dockerfile/test/csharp_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_arm64@sha256:e1345c81aaab856eab0635ddbe612294bee9ee38d4938d3434eab277de6029b8", |
||||
"tools/dockerfile/test/csharp_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/csharp_debian11_x64@sha256:a38ffe41c25486ad0624f54b4a5fa11e74772a06f537c553b3ae3944511ef348", |
||||
"tools/dockerfile/test/cxx_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_alpine_x64@sha256:61fc7408e1171d9470bdd6920cc9da34e31fc43115b80f0fb6f7b9669ba6e366", |
||||
"tools/dockerfile/test/cxx_clang_15_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_clang_15_x64@sha256:aaac47bdeccfcf43331963a75df6a377923c69d1b57ea076c2072b140e00af65", |
||||
"tools/dockerfile/test/cxx_clang_6_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_clang_6_x64@sha256:79ecf682702190564c41289ffe00d4e6f80c104807cca324340349e84288ad99", |
||||
"tools/dockerfile/test/cxx_debian11_openssl102_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian11_openssl102_x64@sha256:8552c41ecca59e32cb3079981cce0b2993a443f1730562a7f19a172ab29f1b2d", |
||||
"tools/dockerfile/test/cxx_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian11_x64@sha256:f4d2b360e8a49d95e8e92971566674a06015427c2488a841b3386feb41d2ff22", |
||||
"tools/dockerfile/test/cxx_debian11_x86.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian11_x86@sha256:77a0be06797567ad9e4924bb5f1a523cd23555af0518a1525fc4a940d60d035c", |
||||
"tools/dockerfile/test/cxx_debian12_openssl309_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_debian12_openssl309_x64@sha256:e8878bf31f42f8af6a7f3b45d0cb79f29fc46c44721b4a8357a2070ab1f5b3e0", |
||||
"tools/dockerfile/test/cxx_gcc_12_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_gcc_12_x64@sha256:ca86af6cb592b4426585a67c7fe58d9925a6e5413801ab45831cd268102c4211", |
||||
"tools/dockerfile/test/cxx_gcc_7_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/cxx_gcc_7_x64@sha256:7d1af94c7329b6b09f6266a56380c0690a31e9121abc89cb8a57820e0f6eb3bb", |
||||
"tools/dockerfile/test/php73_zts_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php73_zts_debian11_x64@sha256:db46a738bf187ffcabbd278a716930c87f90dec2599349c5a52937f9a17e96f8", |
||||
"tools/dockerfile/test/php7_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_arm64@sha256:444e25f9e3a89c2438e4d5e6f3904c5a1f4d1fb961f8456333ebe3e36e301a4e", |
||||
"tools/dockerfile/test/php7_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/php7_debian11_x64@sha256:018d422abf144fc93e9027fd994f7d6aab453fffbe4a669d622dd3d1c1fe9ee7", |
||||
"tools/dockerfile/test/python_alpine_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_alpine_x64@sha256:d10159225ae25276b7ae7bfc4230150e4b0a8ce7be833d904bdd4ecdfdc91c6e", |
||||
"tools/dockerfile/test/python_debian11_default_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_arm64@sha256:868cfb50e465f086b75bb65a7fab6d15b1edefabcd8c1826340acefb6ea1737f", |
||||
"tools/dockerfile/test/python_debian11_default_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/python_debian11_default_x64@sha256:4f29e539941d22b7abb911f9b6b3101ff5c7c4fb75585bfe3b7389251ea6be1d", |
||||
"tools/dockerfile/test/rbe_ubuntu2004.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/rbe_ubuntu2004@sha256:d3951aeadf43e3bee6adc5b86d26cdaf0b9d1b5baf790d7b2530d1c197adc9f8", |
||||
"tools/dockerfile/test/ruby_debian11_arm64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_arm64@sha256:9503d80a40555aba4dd531b64354ad8036c6b37e162c93e7994ca23d89bc7d41", |
||||
"tools/dockerfile/test/ruby_debian11_x64.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/ruby_debian11_x64@sha256:3f01369c3e5707fa63007820b30461b9e32a4b729d81cb92d19669d7966a8584", |
||||
"tools/dockerfile/test/sanity.current_version": "docker://us-docker.pkg.dev/grpc-testing/testing-images-public/sanity@sha256:189e07d8503aa15344e3c8f565783659c3e2edc5b8ca455ec427de1e29ef4504", |
||||
} |
||||
@ -0,0 +1,81 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Generates dockerimage_current_versions.bzl file from the .current_version |
||||
# files found throughout the repository. |
||||
|
||||
set -e |
||||
|
||||
cd $(dirname $0)/../.. |
||||
|
||||
OUTPUT_FILE=tools/bazelify_tests/dockerimage_current_versions.bzl |
||||
if [ "${CHECK_MODE}" != "" ] |
||||
then |
||||
# generate into temporary file instead |
||||
ORIG_FILE="${OUTPUT_FILE}" |
||||
OUTPUT_FILE="$(mktemp)" |
||||
fi |
||||
|
||||
# Generate license header and module docstring. |
||||
cat >"${OUTPUT_FILE}" << EOF |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
""" |
||||
This file is generated by generate_dockerimage_current_versions_bzl.sh |
||||
It makes the info from testing docker image *.current_version files |
||||
accessible to bazel builds. |
||||
""" |
||||
|
||||
EOF |
||||
|
||||
echo "DOCKERIMAGE_CURRENT_VERSIONS = {" >>"${OUTPUT_FILE}" |
||||
|
||||
for current_version_file in $(find tools/ third_party/rake-compiler-dock -name '*.current_version' | LC_ALL=C sort) |
||||
do |
||||
# Remove the docker image tag and only keep the SHA256 digest for the dockerimage. |
||||
# For some reason, bazel's "container-image" exec property doesn't accept image spec |
||||
# with both tag and SHA256 digest. |
||||
echo " \"${current_version_file}\": \"docker://$(cat ${current_version_file} | sed 's/:[a-f0-9]*@sha256/@sha256/')\"," >>"${OUTPUT_FILE}" |
||||
done |
||||
|
||||
echo "}" >>"${OUTPUT_FILE}" |
||||
|
||||
if [ "${CHECK_MODE}" != "" ] |
||||
then |
||||
echo "Checking that ${ORIG_FILE} is up-to-date." |
||||
diff "${OUTPUT_FILE}" "${ORIG_FILE}" || CHECK_FAILED=true |
||||
|
||||
if [ "${CHECK_FAILED}" != "" ] |
||||
then |
||||
echo "CHECK FAILED: Generated file ${ORIG_FILE} is out of date and needs to be regenerated." |
||||
exit 1 |
||||
else |
||||
echo "${ORIG_FILE} is up-to-date." |
||||
fi |
||||
else |
||||
echo "Generated ${OUTPUT_FILE}" |
||||
fi |
||||
@ -0,0 +1,122 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -e |
||||
|
||||
ARCHIVE_WO_SUBMODULES="$1" |
||||
ARCHIVE_WITH_SUBMODULES="$2" |
||||
export ARCHIVES_DIR="$(pwd)/archives" |
||||
|
||||
export ARCHIVE_FORMAT=tar |
||||
|
||||
mkdir -p "${ARCHIVES_DIR}" |
||||
rm -rf "${ARCHIVES_DIR}"/* |
||||
|
||||
# TODO(jtattermusch): This script is currently only tested on linux. |
||||
# Nothing prevents it from working on other systems in principle, |
||||
# but more work is needed. |
||||
|
||||
# HACK: To be able to collect all grpc source files as an archive |
||||
# we need to break from bazel's "sandbox" to be able to read files |
||||
# from the original bazel workspace (which in our case is the grpc repository root) |
||||
# This action runs with the "standalone" (a.k.a) local strategy, |
||||
# so path to the original bazel workspace from where this was invoked |
||||
# can be obtained by resolving the link that points to one of the |
||||
# source files. |
||||
# 1. find first component of the relative path to this script |
||||
# 2. resolve the symlink (it will point to same dir in the workspace) |
||||
# 3. one level up is the root of the original bazel workspace |
||||
FIRST_PATH_COMPONENT="$(echo $0 | sed 's|/.*||')" |
||||
ORIGINAL_BAZEL_WORKSPACE_ROOT="$(dirname $(readlink ${FIRST_PATH_COMPONENT}))" |
||||
|
||||
# extract STABLE_GIT_COMMIT from stable-status.txt |
||||
GRPC_GIT_COMMIT_FROM_STABLE_STATUS=$(grep ^STABLE_GRPC_GIT_COMMIT bazel-out/stable-status.txt | cut -d' ' -f2) |
||||
|
||||
if [ "${GRPC_GIT_COMMIT_FROM_STABLE_STATUS}" == "" ] |
||||
then |
||||
echo "Failed to obtain info about gRPC git commit from the bazel workspace. Make sure you invoke bazel with --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh" >&2 |
||||
exit 1 |
||||
fi |
||||
|
||||
GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS=$(grep ^STABLE_GRPC_UNCOMMITED_PATCH_CHECKSUM bazel-out/stable-status.txt | cut -d' ' -f2) |
||||
GRPC_GIT_WORKSPACE_DIRTY_FROM_STABLE_STATUS=$(grep ^STABLE_GRPC_GIT_WORKSPACE_DIRTY bazel-out/stable-status.txt | cut -d' ' -f2) |
||||
|
||||
pushd ${ORIGINAL_BAZEL_WORKSPACE_ROOT} >/dev/null |
||||
|
||||
if [ "${GRPC_GIT_COMMIT_FROM_STABLE_STATUS}" != "$(git rev-parse HEAD)" ] |
||||
then |
||||
echo "Info about gRPC commit from stable-status.txt doesn't match the commit at which the workspace root is." >&2 |
||||
echo "This is unexpected and giving up early is better than risking to end up with bogus results." >&2 |
||||
exit 1 |
||||
fi |
||||
|
||||
mkdir -p ${ARCHIVES_DIR}/grpc |
||||
git archive --format="${ARCHIVE_FORMAT}" HEAD >"${ARCHIVES_DIR}/grpc/$(git rev-parse HEAD).${ARCHIVE_FORMAT}" |
||||
|
||||
if [ "${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}" != "" ] |
||||
then |
||||
git diff HEAD >"${ARCHIVES_DIR}/grpc/grpc_uncommited_${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}.patch" |
||||
# check that the actual checksum of the patch file is what we expect it to be |
||||
echo "${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS} ${ARCHIVES_DIR}/grpc/grpc_uncommited_${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}.patch" | sha256sum --quiet --check |
||||
fi |
||||
|
||||
# produce archive for each submodule |
||||
git submodule --quiet foreach 'git_commit="$(git rev-parse HEAD)"; mkdir -p ${ARCHIVES_DIR}/${name}; git archive --format=${ARCHIVE_FORMAT} HEAD >${ARCHIVES_DIR}/${name}/${git_commit}.${ARCHIVE_FORMAT}' |
||||
|
||||
popd >/dev/null |
||||
|
||||
# Extract grpc |
||||
mkdir grpc |
||||
tar -xopf "${ARCHIVES_DIR}/grpc/${GRPC_GIT_COMMIT_FROM_STABLE_STATUS}.${ARCHIVE_FORMAT}" -C grpc |
||||
|
||||
# apply the patch |
||||
if [ "${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}" != "" ] |
||||
then |
||||
pushd grpc >/dev/null |
||||
patch --quiet -p1 <"${ARCHIVES_DIR}/grpc/grpc_uncommited_${GRPC_UNCOMMITED_PATCH_CHECKSUM_FROM_STABLE_STATUS}.patch" |
||||
popd >/dev/null |
||||
fi |
||||
|
||||
# The archive produced need to be deterministic/stable. |
||||
# Passing the following args to tar should be enough to make them so. |
||||
# See https://reproducible-builds.org/docs/archives/ |
||||
DETERMINISTIC_TAR_ARGS=( |
||||
--sort=name |
||||
# use a fixed mtime timestamp for all files (2015-01-01 00:00Z works just fine) |
||||
--mtime="@1420070400" |
||||
--owner=0 |
||||
--group=0 |
||||
--numeric-owner |
||||
--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime |
||||
) |
||||
|
||||
# create archive without submodules first |
||||
tar "${DETERMINISTIC_TAR_ARGS[@]}" -czf ${ARCHIVE_WO_SUBMODULES} grpc |
||||
|
||||
SUBMODULE_ARCHIVE_LIST="$(grep 'STABLE_GRPC_SUBMODULE_ARCHIVES ' bazel-out/stable-status.txt | sed 's/^STABLE_GRPC_SUBMODULE_ARCHIVES //')" |
||||
# TODO(jtattermusch): handle spaces in submodule directory path |
||||
for submodule_archive in ${SUBMODULE_ARCHIVE_LIST} |
||||
do |
||||
archive_subdir="grpc/$(dirname ${submodule_archive})" |
||||
mkdir -p $archive_subdir |
||||
# Extract submodule archive under the correct subdirectory in grpc |
||||
tar -xopf "${ARCHIVES_DIR}/${submodule_archive}.${ARCHIVE_FORMAT}" -C $archive_subdir |
||||
done |
||||
|
||||
# create archive with everything |
||||
tar "${DETERMINISTIC_TAR_ARGS[@]}" -czf ${ARCHIVE_WITH_SUBMODULES} grpc |
||||
|
||||
# Cleanup intermediate files we created |
||||
rm -rf "${ARCHIVES_DIR}" grpc |
||||
@ -0,0 +1,54 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
if [ "${GRPC_RUNTESTS_USE_LOGIN_SHELL}" != "" ] |
||||
then |
||||
unset GRPC_RUNTESTS_USE_LOGIN_SHELL |
||||
# respawn the entire script using login shell |
||||
exec bash -l "$0" "$@" |
||||
fi |
||||
|
||||
ARCHIVE_WITH_SUBMODULES="$1" |
||||
shift |
||||
|
||||
# The JUnit XML report file generated by run_tests.py is compatible with |
||||
# the report format accepted by bazel as the result for tests target. |
||||
REPORT_XML_FILE="${XML_OUTPUT_FILE}" |
||||
# Create report suite name from the last component of the bazel target's name. |
||||
REPORT_SUITE_NAME="$(echo ${TEST_TARGET} | sed 's|^.*[:/]||')" |
||||
|
||||
# Extract grpc repo archive |
||||
tar -xopf ${ARCHIVE_WITH_SUBMODULES} |
||||
cd grpc |
||||
|
||||
if [ "${GRPC_RUNTESTS_PREPARE_SCRIPT}" != "" ] |
||||
then |
||||
source "../${GRPC_RUNTESTS_PREPARE_SCRIPT}" |
||||
fi |
||||
|
||||
python3 tools/run_tests/run_tests.py -t -j "$(nproc)" -x "${REPORT_XML_FILE}" --report_suite_name "${REPORT_SUITE_NAME}" "$@" || FAILED="true" |
||||
|
||||
if [ -x "$(command -v ccache)" ] |
||||
then |
||||
ccache --show-stats || true |
||||
fi |
||||
|
||||
if [ "$FAILED" != "" ] |
||||
then |
||||
exit 1 |
||||
fi |
||||
|
||||
@ -0,0 +1,158 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
load("//bazel:grpc_build_system.bzl", "grpc_package") |
||||
load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") |
||||
load(":portability_tests.bzl", "generate_run_tests_portability_tests") |
||||
|
||||
licenses(["notice"]) |
||||
|
||||
grpc_package(name = "tools/bazelify_tests/test") |
||||
|
||||
generate_run_tests_portability_tests(name = "portability_tests_linux") |
||||
|
||||
# C/C++ |
||||
grpc_run_tests_py_test( |
||||
name = "runtests_c_linux_dbg", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l c -c dbg", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", |
||||
) |
||||
|
||||
grpc_run_tests_py_test( |
||||
name = "runtests_c_linux_opt", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l c -c opt", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", |
||||
) |
||||
|
||||
grpc_run_tests_py_test( |
||||
name = "runtests_cpp_linux_dbg_build_only", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l c++ -c dbg --build_only", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", |
||||
) |
||||
|
||||
grpc_run_tests_py_test( |
||||
name = "runtests_cpp_linux_opt_build_only", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l c++ -c opt --build_only", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", |
||||
) |
||||
|
||||
# TODO(jtattermusch): Reintroduce ruby tests once they pass. |
||||
# # Ruby |
||||
# grpc_run_tests_py_test( |
||||
# name = "runtests_ruby_linux_dbg", |
||||
# size = "enormous", |
||||
# args = [ |
||||
# "-l ruby -c dbg", |
||||
# ], |
||||
# docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version", |
||||
# prepare_script = ":prepare_ruby.sh", |
||||
# use_login_shell = True, # ruby's docker image uses RVM which wierdly requires login shell |
||||
# ) |
||||
|
||||
# grpc_run_tests_py_test( |
||||
# name = "runtests_ruby_linux_opt", |
||||
# size = "enormous", |
||||
# args = [ |
||||
# "-l ruby -c opt", |
||||
# ], |
||||
# docker_image_version = "tools/dockerfile/test/ruby_debian11_x64.current_version", |
||||
# prepare_script = ":prepare_ruby.sh", |
||||
# use_login_shell = True, # ruby's docker image uses RVM which wierdly requires login shell |
||||
# ) |
||||
|
||||
# PHP |
||||
grpc_run_tests_py_test( |
||||
name = "runtests_php_linux_dbg", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l php7 -c dbg", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version", |
||||
) |
||||
|
||||
grpc_run_tests_py_test( |
||||
name = "runtests_php_linux_opt", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l php7 -c opt", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/php7_debian11_x64.current_version", |
||||
) |
||||
|
||||
# TODO(jtattermusch): Reintroduce python tests once they pass. |
||||
# # Python |
||||
# grpc_run_tests_py_test( |
||||
# name = "runtests_python_linux_opt", |
||||
# size = "enormous", |
||||
# args = [ |
||||
# "-l python -c opt", |
||||
# ], |
||||
# docker_image_version = "tools/dockerfile/test/python_debian11_default_x64.current_version", |
||||
# ) |
||||
|
||||
# C# |
||||
grpc_run_tests_py_test( |
||||
name = "runtests_csharp_linux_dbg", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l csharp -c dbg", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/csharp_debian11_x64.current_version", |
||||
) |
||||
|
||||
grpc_run_tests_py_test( |
||||
name = "runtests_csharp_linux_opt", |
||||
size = "enormous", |
||||
args = [ |
||||
"-l csharp -c opt", |
||||
], |
||||
docker_image_version = "tools/dockerfile/test/csharp_debian11_x64.current_version", |
||||
) |
||||
|
||||
test_suite( |
||||
name = "basic_tests_linux", |
||||
tests = [ |
||||
":runtests_c_linux_dbg", |
||||
":runtests_c_linux_opt", |
||||
":runtests_cpp_linux_dbg_build_only", |
||||
":runtests_cpp_linux_opt_build_only", |
||||
":runtests_csharp_linux_dbg", |
||||
":runtests_csharp_linux_opt", |
||||
":runtests_php_linux_dbg", |
||||
":runtests_php_linux_opt", |
||||
#":runtests_python_linux_opt", |
||||
#":runtests_ruby_linux_dbg", |
||||
#":runtests_ruby_linux_opt", |
||||
], |
||||
) |
||||
|
||||
test_suite( |
||||
name = "all_tests_linux", |
||||
tests = [ |
||||
":basic_tests_linux", |
||||
":portability_tests_linux", |
||||
], |
||||
) |
||||
@ -0,0 +1,91 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
""" |
||||
Generates portability tests. |
||||
""" |
||||
|
||||
load("//tools/bazelify_tests:build_defs.bzl", "grpc_run_tests_py_test") |
||||
|
||||
def _safe_language_name(name): |
||||
"""Character '+' isn't allowed in bazel target name""" |
||||
return name.replace("+", "p") |
||||
|
||||
def generate_run_tests_portability_tests(name): |
||||
"""Generates run_tests_py portability test targets. |
||||
|
||||
Args: |
||||
name: Name of the test suite that will be generated. |
||||
""" |
||||
test_names = [] |
||||
|
||||
# portability C x86 |
||||
grpc_run_tests_py_test( |
||||
name = "runtests_c_linux_dbg_x86", |
||||
args = ["-l c -c dbg"], |
||||
docker_image_version = "tools/dockerfile/test/cxx_debian11_x86.current_version", |
||||
size = "enormous", |
||||
) |
||||
test_names.append("runtests_c_linux_dbg_x86") |
||||
|
||||
# C and C++ with no-exceptions on Linux |
||||
for language in ["c", "c++"]: |
||||
test_name = "runtests_%s_linux_dbg_noexcept_build_only" % _safe_language_name(language) |
||||
grpc_run_tests_py_test( |
||||
name = test_name, |
||||
args = ["-l %s --config noexcept --build_only" % language], |
||||
docker_image_version = "tools/dockerfile/test/cxx_debian11_x64.current_version", |
||||
size = "enormous", |
||||
) |
||||
test_names.append(test_name) |
||||
|
||||
# C and C++ under different compilers |
||||
for language in ["c", "c++"]: |
||||
compiler_configs = [ |
||||
# TODO(b/283304471): Add 'gcc10.2_openssl102' once possible |
||||
["gcc_7", "", "tools/dockerfile/test/cxx_gcc_7_x64.current_version"], |
||||
["gcc_12", "--cmake_configure_extra_args=-DCMAKE_CXX_STANDARD=20", "tools/dockerfile/test/cxx_gcc_12_x64.current_version"], |
||||
# TODO(jtattermusch): Re-enable once the build can finish in reasonable time (looks like ccache is not being used?) |
||||
#["gcc_musl", "", "tools/dockerfile/test/cxx_alpine_x64.current_version"], |
||||
["clang_6", "--cmake_configure_extra_args=-DCMAKE_C_COMPILER=clang --cmake_configure_extra_args=-DCMAKE_CXX_COMPILER=clang++", "tools/dockerfile/test/cxx_clang_6_x64.current_version"], |
||||
["clang_15", "--cmake_configure_extra_args=-DCMAKE_C_COMPILER=clang --cmake_configure_extra_args=-DCMAKE_CXX_COMPILER=clang++", "tools/dockerfile/test/cxx_clang_15_x64.current_version"], |
||||
] |
||||
|
||||
for compiler_name, args, docker_image_version in compiler_configs: |
||||
test_name = "runtests_%s_linux_dbg_%s_build_only" % (_safe_language_name(language), compiler_name) |
||||
grpc_run_tests_py_test( |
||||
name = test_name, |
||||
args = ["-l %s -c dbg %s --build_only" % (language, args)], |
||||
docker_image_version = docker_image_version, |
||||
size = "enormous", |
||||
) |
||||
test_names.append(test_name) |
||||
|
||||
# TODO(jtattermusch): Reintroduce the test once it passes. |
||||
# Python on alpine |
||||
#grpc_run_tests_py_test( |
||||
# name = "runtests_python_linux_dbg_alpine", |
||||
# args = [ |
||||
# "-l python -c dbg --compiler python_alpine", |
||||
# ], |
||||
# docker_image_version = "tools/dockerfile/test/python_alpine_x64.current_version", |
||||
# size = "enormous", |
||||
#) |
||||
#test_names.append("runtests_python_linux_dbg_alpine") |
||||
|
||||
# Generate test suite that allows easily running all portability tests. |
||||
native.test_suite( |
||||
name = name, |
||||
tests = [(":%s" % test_name) for test_name in test_names], |
||||
) |
||||
@ -0,0 +1,36 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -e |
||||
|
||||
GRPC_GIT_COMMIT="$(git rev-parse HEAD)" |
||||
echo "STABLE_GRPC_GIT_COMMIT ${GRPC_GIT_COMMIT}" |
||||
|
||||
# produce a value that has name and commit for every submodule |
||||
echo -n 'STABLE_GRPC_SUBMODULE_ARCHIVES' |
||||
git submodule --quiet foreach 'git_commit="$(git rev-parse HEAD)"; echo -n " ${name}/${git_commit}"'; echo "" |
||||
|
||||
# set info about whether the git workspace is clean/dirty and checksum of git patch |
||||
if [ "$(git status --porcelain)" == "" ] |
||||
then |
||||
echo "STABLE_GRPC_GIT_WORKSPACE_DIRTY false" |
||||
else |
||||
echo "STABLE_GRPC_GIT_WORKSPACE_DIRTY true" |
||||
echo "STABLE_GRPC_UNCOMMITED_PATCH_CHECKSUM $(git diff HEAD | sha256sum | cut -f1 -d' ')" |
||||
fi |
||||
|
||||
# Since only --workspace_status_command is allowed by bazel, also include |
||||
# status from the "default" workspace status command. |
||||
tools/remote_build/workspace_status_kokoro.sh |
||||
@ -0,0 +1,52 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Config file for the internal CI (in protobuf text format) |
||||
|
||||
# Location of the continuous shell script in repository. |
||||
build_file: "grpc/tools/internal_ci/linux/grpc_bazel_rbe.sh" |
||||
timeout_mins: 90 |
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*sponge_log.*" |
||||
regex: "github/grpc/reports/**" |
||||
} |
||||
} |
||||
|
||||
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/resultstore_api_key" |
||||
|
||||
bazel_setting { |
||||
# In order for Kokoro to recognize this as a bazel build and publish the bazel resultstore link, |
||||
# the bazel_setting section needs to be present and "upsalite_frontend_address" needs to be |
||||
# set. The rest of configuration from bazel_setting is unused (we configure everything when bazel |
||||
# command is invoked). |
||||
upsalite_frontend_address: "https://source.cloud.google.com" |
||||
} |
||||
|
||||
env_vars { |
||||
# flags will be passed to bazel invocation |
||||
key: "BAZEL_FLAGS" |
||||
value: "--genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh --cache_test_results=no" |
||||
} |
||||
|
||||
env_vars { |
||||
# tests to run by the bazel invocation |
||||
key: "BAZEL_TESTS" |
||||
value: "//tools/bazelify_tests/test:all_tests_linux" |
||||
} |
||||
|
||||
env_vars { |
||||
key: "UPLOAD_TEST_RESULTS" |
||||
value: "true" |
||||
} |
||||
@ -0,0 +1,47 @@ |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# Config file for the internal CI (in protobuf text format) |
||||
|
||||
# Location of the continuous shell script in repository. |
||||
build_file: "grpc/tools/internal_ci/linux/grpc_bazel_rbe.sh" |
||||
timeout_mins: 90 |
||||
action { |
||||
define_artifacts { |
||||
regex: "**/*sponge_log.*" |
||||
regex: "github/grpc/reports/**" |
||||
} |
||||
} |
||||
|
||||
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/resultstore_api_key" |
||||
|
||||
bazel_setting { |
||||
# In order for Kokoro to recognize this as a bazel build and publish the bazel resultstore link, |
||||
# the bazel_setting section needs to be present and "upsalite_frontend_address" needs to be |
||||
# set. The rest of configuration from bazel_setting is unused (we configure everything when bazel |
||||
# command is invoked). |
||||
upsalite_frontend_address: "https://source.cloud.google.com" |
||||
} |
||||
|
||||
env_vars { |
||||
# flags will be passed to bazel invocation |
||||
key: "BAZEL_FLAGS" |
||||
value: "--genrule_strategy=remote,local --workspace_status_command=tools/bazelify_tests/workspace_status_cmd.sh" |
||||
} |
||||
|
||||
env_vars { |
||||
# tests to run by the bazel invocation |
||||
key: "BAZEL_TESTS" |
||||
value: "//tools/bazelify_tests/test:all_tests_linux" |
||||
} |
||||
@ -0,0 +1,45 @@ |
||||
#@IgnoreInspection BashAddShebang |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
# bazelrc file for running gRPC tests in a docker sandbox to help |
||||
# with debugging issues with RBE. |
||||
# TODO(jtattermusch): Settings in this file are currently EXPERIMENTAL. Use |
||||
# at your own risk. |
||||
|
||||
# TODO(jtattermusch): Consider including some configuration settings from |
||||
# tools/remote_build/include/rbe_remote_execution.bazelrc |
||||
|
||||
build --experimental_docker_verbose |
||||
build --experimental_enable_docker_sandbox |
||||
|
||||
build --spawn_strategy=docker |
||||
build --strategy=Javac=docker |
||||
build --strategy=Closure=docker |
||||
build --genrule_strategy=docker |
||||
|
||||
# Next section is linux-specific RBE configuration |
||||
build --crosstool_top=//third_party/toolchains:rbe_linux_default_toolchain_suite |
||||
build --extra_toolchains=//third_party/toolchains:rbe_linux_default_cc_toolchain |
||||
# Use custom execution platforms defined in third_party/toolchains |
||||
build --extra_execution_platforms=//third_party/toolchains:rbe_linux_default_platform |
||||
build --host_platform=//third_party/toolchains:rbe_linux_default_platform |
||||
build --platforms=//third_party/toolchains:rbe_linux_default_platform |
||||
|
||||
# we assume the default bazel RBE build is on linux, |
||||
# so filter out stuff that should not be built or run there. |
||||
build --test_tag_filters=-no_linux |
||||
build --build_tag_filters=-no_linux |
||||
|
||||
import %workspace%/tools/remote_build/include/test_config_common.bazelrc |
||||
Loading…
Reference in new issue