diff --git a/tools/internal_ci/linux/arm64/grpc_bazel.sh b/tools/internal_ci/linux/arm64/grpc_bazel.sh index 0503692aea0..60182ac7711 100755 --- a/tools/internal_ci/linux/arm64/grpc_bazel.sh +++ b/tools/internal_ci/linux/arm64/grpc_bazel.sh @@ -15,11 +15,13 @@ set -ex +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + # change to grpc repo root cd $(dirname $0)/../../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc export DOCKERFILE_DIR=tools/dockerfile/test/bazel_arm64 -export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT -exec tools/run_tests/dockerize/build_and_run_docker.sh +exec tools/run_tests/dockerize/build_and_run_docker.sh "${BAZEL_SCRIPT}" diff --git a/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp.cfg b/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp.cfg index 38308432826..2511409ceba 100644 --- a/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp.cfg +++ b/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp.cfg @@ -17,6 +17,13 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/arm64/grpc_bazel.sh" timeout_mins: 120 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh" diff --git a/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh b/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh index 284f4bf1740..615257c134d 100755 --- a/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh +++ b/tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh @@ -15,13 +15,6 @@ set -ex -mkdir -p /var/local/git -git clone /var/local/jenkins/grpc /var/local/git/grpc -(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ -${name}') -cd /var/local/git/grpc - # tests require port server to be running python3 tools/run_tests/start_port_server.py diff --git a/tools/internal_ci/linux/grpc_bazel.sh b/tools/internal_ci/linux/grpc_bazel.sh index 2b17fa849a3..5f6603c7e1b 100755 --- a/tools/internal_ci/linux/grpc_bazel.sh +++ b/tools/internal_ci/linux/grpc_bazel.sh @@ -15,11 +15,13 @@ set -ex +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + # change to grpc repo root cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc export DOCKERFILE_DIR=tools/dockerfile/test/bazel -export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT -exec tools/run_tests/dockerize/build_and_run_docker.sh +exec tools/run_tests/dockerize/build_and_run_docker.sh "${BAZEL_SCRIPT}" diff --git a/tools/internal_ci/linux/grpc_bazel_build.cfg b/tools/internal_ci/linux/grpc_bazel_build.cfg index 59f5da9a678..44fd1d7bee0 100644 --- a/tools/internal_ci/linux/grpc_bazel_build.cfg +++ b/tools/internal_ci/linux/grpc_bazel_build.cfg @@ -17,6 +17,13 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 60 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/grpc_bazel_build_in_docker.sh" diff --git a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh index 77fb993124c..71b585a8fc1 100755 --- a/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh +++ b/tools/internal_ci/linux/grpc_bazel_build_in_docker.sh @@ -12,38 +12,34 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Test basic Bazel features -# -# NOTE: No empty lines should appear in this file before igncr is set! -set -ex -o igncr || set -ex - -mkdir -p /var/local/git -git clone /var/local/jenkins/grpc /var/local/git/grpc -(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ -${name}') -cd /var/local/git/grpc # Build all basic targets using the strict warning option which leverages the # clang compiler to check if sources can pass a set of warning options. # For now //examples/android/binder/ are excluded because it needs Android # SDK/NDK to be installed to build -bazel build --define=use_strict_warning=true \ - -- \ - :all \ - //src/core/... \ - //src/compiler/... \ - //test/... \ - //examples/... \ - -//examples/android/binder/... +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_build_with_strict_warnings +bazel_build_with_strict_warnings/bazel_wrapper \ + --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \ + build \ + --define=use_strict_warning=true \ + -- \ + :all \ + //src/core/... \ + //src/compiler/... \ + //test/... \ + //examples/... \ + -//examples/android/binder/... # TODO(veblush): Remove this test after migration to abseil-status is done. -bazel build --define=use_strict_warning=true --define=use_abseil_status=true \ - -- \ - //src/core/... \ - //src/compiler/... \ - //test/... +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_build_with_abseil_status +bazel_build_with_abseil_status/bazel_wrapper \ + --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \ + build \ + --define=use_strict_warning=true --define=use_abseil_status=true \ + -- \ + //src/core/... \ + //src/compiler/... \ + //test/... # TODO(jtattersmusch): Adding a build here for --define=grpc_no_xds is not ideal # and we should find a better place for this. Refer @@ -56,6 +52,6 @@ EXIT_CODE=0 bazel build //test/cpp/end2end/xds:xds_end2end_test --define=grpc_no_xds=true || EXIT_CODE=$? if [ $EXIT_CODE -eq 0 ] then - echo "Building xds_end2end_test succeeded even with --define=grpc_no_xds=true" - exit 1 + echo "Building xds_end2end_test succeeded even with --define=grpc_no_xds=true" + exit 1 fi diff --git a/tools/internal_ci/linux/grpc_binder_transport_apk.cfg b/tools/internal_ci/linux/grpc_binder_transport_apk.cfg index 1c2b38c61b8..0021005e88f 100644 --- a/tools/internal_ci/linux/grpc_binder_transport_apk.cfg +++ b/tools/internal_ci/linux/grpc_binder_transport_apk.cfg @@ -17,6 +17,13 @@ # Location of the continuous shell script in repository. build_file: "grpc/tools/internal_ci/linux/grpc_binder_transport_apk.sh" timeout_mins: 60 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } +} + env_vars { key: "BAZEL_SCRIPT" value: "tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh" diff --git a/tools/internal_ci/linux/grpc_binder_transport_apk.sh b/tools/internal_ci/linux/grpc_binder_transport_apk.sh index d85474a5fa7..1a27fa24883 100755 --- a/tools/internal_ci/linux/grpc_binder_transport_apk.sh +++ b/tools/internal_ci/linux/grpc_binder_transport_apk.sh @@ -15,11 +15,13 @@ set -ex +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + # change to grpc repo root cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc export DOCKERFILE_DIR=tools/dockerfile/test/binder_transport_apk -export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT -exec tools/run_tests/dockerize/build_and_run_docker.sh +exec tools/run_tests/dockerize/build_and_run_docker.sh "${BAZEL_SCRIPT}" diff --git a/tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh b/tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh old mode 100644 new mode 100755 index 454e2fe1fd8..d368826e9a9 --- a/tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh +++ b/tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # Copyright 2021 gRPC authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -12,17 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -#!/usr/bin/env bash -# -# NOTE: No empty lines should appear in this file before igncr is set! -set -ex -o igncr || set -ex - -mkdir -p /var/local/git -git clone /var/local/jenkins/grpc /var/local/git/grpc -(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ -${name}') -cd /var/local/git/grpc +set -ex echo $ANDROID_HOME echo $ANDROID_NDK_HOME @@ -32,7 +23,11 @@ echo $ANDROID_NDK_HOME # CPU are specified because gRPC does not build with 32bit NDK (which has socklen_t # defined as int due to an accident). # The python option is for disabling python2 enforcement when packing APK -bazel build --define=use_strict_warning=true \ +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_binder_example_app +bazel_binder_example_app/bazel_wrapper \ + --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \ + build \ + --define=use_strict_warning=true \ --fat_apk_cpu=x86_64,arm64-v8a \ --extra_toolchains=@rules_python//python:autodetecting_toolchain_nonstrict \ //examples/android/binder/java/io/grpc/binder/cpp/exampleclient:app \ @@ -40,5 +35,9 @@ bazel build --define=use_strict_warning=true \ # Make sure the Java code that will be invoked by binder transport # implementation builds -bazel build --define=use_strict_warning=true \ +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_binder_connection_helper +bazel_binder_connection_helper/bazel_wrapper \ + --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \ + build \ + --define=use_strict_warning=true \ @binder_transport_android_helper//io/grpc/binder/cpp:connection_helper diff --git a/tools/internal_ci/linux/grpc_feature_example_tests.cfg b/tools/internal_ci/linux/grpc_feature_example_tests.cfg index a23887a33f5..4eea19c6291 100644 --- a/tools/internal_ci/linux/grpc_feature_example_tests.cfg +++ b/tools/internal_ci/linux/grpc_feature_example_tests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 30 env_vars { key: "BAZEL_SCRIPT" - value: "tools/internal_ci/linux/grpc_feature_example_tests.sh" + value: "tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh" } action { define_artifacts { diff --git a/tools/internal_ci/linux/grpc_feature_example_tests.sh b/tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh similarity index 62% rename from tools/internal_ci/linux/grpc_feature_example_tests.sh rename to tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh index 0103cf4e2c8..998a28a361d 100755 --- a/tools/internal_ci/linux/grpc_feature_example_tests.sh +++ b/tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh @@ -12,17 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -#!/usr/bin/env bash -# -# NOTE: No empty lines should appear in this file before igncr is set! -set -ex -o igncr || set -ex -mkdir -p /var/local/git -git clone /var/local/jenkins/grpc /var/local/git/grpc -(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ -${name}') -cd /var/local/git/grpc +set -ex apt-get install -y lsof diff --git a/tools/internal_ci/linux/grpc_flaky_network.cfg b/tools/internal_ci/linux/grpc_flaky_network.cfg index 07bedd79f94..5ba4dd208cc 100644 --- a/tools/internal_ci/linux/grpc_flaky_network.cfg +++ b/tools/internal_ci/linux/grpc_flaky_network.cfg @@ -15,9 +15,11 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/linux/grpc_bazel_privileged_docker.sh" +build_file: "grpc/tools/internal_ci/linux/grpc_flaky_network.sh" timeout_mins: 240 -env_vars { - key: "BAZEL_SCRIPT" - value: "tools/internal_ci/linux/grpc_flaky_network_in_docker.sh" +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + } } diff --git a/tools/internal_ci/linux/grpc_bazel_privileged_docker.sh b/tools/internal_ci/linux/grpc_flaky_network.sh similarity index 72% rename from tools/internal_ci/linux/grpc_bazel_privileged_docker.sh rename to tools/internal_ci/linux/grpc_flaky_network.sh index ae1056d7c3d..0793065bd88 100755 --- a/tools/internal_ci/linux/grpc_bazel_privileged_docker.sh +++ b/tools/internal_ci/linux/grpc_flaky_network.sh @@ -15,12 +15,15 @@ set -ex +# avoid slow finalization after the script has exited. +source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc + # change to grpc repo root cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc export DOCKERFILE_DIR=tools/dockerfile/test/bazel -export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT # NET_ADMIN capability allows tests to manipulate network interfaces -exec tools/run_tests/dockerize/build_and_run_docker.sh --cap-add NET_ADMIN +export EXTRA_DOCKER_ARGS="--cap-add NET_ADMIN" +exec tools/run_tests/dockerize/build_and_run_docker.sh tools/internal_ci/linux/grpc_flaky_network_in_docker.sh diff --git a/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh b/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh index b861627058d..095433bba91 100755 --- a/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh +++ b/tools/internal_ci/linux/grpc_flaky_network_in_docker.sh @@ -15,17 +15,10 @@ # # Run the flaky network test # -# NOTE: No empty lines should appear in this file before igncr is set! -set -ex -o igncr || set -ex - -mkdir -p /var/local/git -git clone /var/local/jenkins/grpc /var/local/git/grpc -(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ -${name}') -cd /var/local/git/grpc/test/cpp/end2end +set -ex # iptables is used to drop traffic between client and server apt-get install -y iptables iproute2 -bazel test --test_output=all --test_timeout=1200 :flaky_network_test --test_env=GRPC_TRACE=http --test_env=GRPC_VERBOSITY=DEBUG +python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_flaky_network_test +bazel_flaky_network_test/bazel_wrapper test --test_output=all --test_timeout=1200 //test/cpp/end2end:flaky_network_test --test_env=GRPC_TRACE=http --test_env=GRPC_VERBOSITY=DEBUG diff --git a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh index a36cd416a96..a67399ae0d7 100755 --- a/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh +++ b/tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh @@ -12,25 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# -# Test full Bazel set -ex -mkdir -p /var/local/git -git clone /var/local/jenkins/grpc /var/local/git/grpc -(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \ -${name}') -cd /var/local/git/grpc +RESULTSTORE_RESULTS_FLAG="--bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc" TEST_TARGETS="//src/python/... //tools/distrib/python/grpcio_tools/... //examples/python/..." BAZEL_FLAGS="--test_output=errors" python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests -python_bazel_tests/bazel_wrapper test ${BAZEL_FLAGS} ${TEST_TARGETS} +python_bazel_tests/bazel_wrapper ${RESULTSTORE_RESULTS_FLAG} test ${BAZEL_FLAGS} ${TEST_TARGETS} python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_single_threaded_unary_streams -python_bazel_tests_single_threaded_unary_streams/bazel_wrapper test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS} +python_bazel_tests_single_threaded_unary_streams/bazel_wrapper ${RESULTSTORE_RESULTS_FLAG} test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS} python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_poller_engine -python_bazel_tests_poller_engine/bazel_wrapper test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS} +python_bazel_tests_poller_engine/bazel_wrapper ${RESULTSTORE_RESULTS_FLAG} test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS} diff --git a/tools/internal_ci/linux/pull_request/grpc_feature_example_tests.cfg b/tools/internal_ci/linux/pull_request/grpc_feature_example_tests.cfg index 1c7eb6b5828..1999e3192e7 100644 --- a/tools/internal_ci/linux/pull_request/grpc_feature_example_tests.cfg +++ b/tools/internal_ci/linux/pull_request/grpc_feature_example_tests.cfg @@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh" timeout_mins: 30 env_vars { key: "BAZEL_SCRIPT" - value: "tools/internal_ci/linux/grpc_feature_example_tests.sh" + value: "tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh" } action { define_artifacts { diff --git a/tools/run_tests/dockerize/docker_propagate_env.list b/tools/run_tests/dockerize/docker_propagate_env.list index 5ea24a2bd2f..250b6c56a43 100644 --- a/tools/run_tests/dockerize/docker_propagate_env.list +++ b/tools/run_tests/dockerize/docker_propagate_env.list @@ -12,6 +12,7 @@ KOKORO_BUILD_INITIATOR KOKORO_BUILD_NUMBER KOKORO_BUILD_URL KOKORO_GIT_COMMIT +KOKORO_GITHUB_COMMIT KOKORO_JOB_NAME KOKORO_GITHUB_PULL_REQUEST_NUMBER KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH