Cleanup & improve bazel-based test jobs running under docker. (#29402)

* use EXTRA_DOCKER_ARGS in grpc_bazel_privileged_docker.sh

* docker_propagate_env.list: propagate all variables needed by workspace_status_kokoro.sh

* simplify bazel invocations in docker

* use respawn script for grpc_bazel.sh

* cleanup grpc_flaky_network test

* cleanup grpc_binder_transport_apk job

* cleanup grpc_feature_example_tests

* additional cleanup & improvements

tools/internal_ci/linux/arm64/grpc_bazel.sh

@@ -15,11 +15,13 @@
 
 set -ex
 
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
 # change to grpc repo root
 cd $(dirname $0)/../../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 export DOCKERFILE_DIR=tools/dockerfile/test/bazel_arm64
-export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT
+exec tools/run_tests/dockerize/build_and_run_docker.sh "${BAZEL_SCRIPT}"
-exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp.cfg

@@ -17,6 +17,13 @@
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/arm64/grpc_bazel.sh"
 timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
+}
+
 env_vars {
   key: "BAZEL_SCRIPT"
   value: "tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh"

tools/internal_ci/linux/arm64/grpc_bazel_test_c_cpp_in_docker.sh

@@ -15,13 +15,6 @@
 
 set -ex
 
-mkdir -p /var/local/git
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
-cd /var/local/git/grpc
-
 # tests require port server to be running
 python3 tools/run_tests/start_port_server.py
 

tools/internal_ci/linux/grpc_bazel.sh

@@ -15,11 +15,13 @@
 
 set -ex
 
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
 # change to grpc repo root
 cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 export DOCKERFILE_DIR=tools/dockerfile/test/bazel
-export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT
+exec tools/run_tests/dockerize/build_and_run_docker.sh "${BAZEL_SCRIPT}"
-exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/grpc_bazel_build.cfg

@@ -17,6 +17,13 @@
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
 timeout_mins: 60
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
+}
+
 env_vars {
   key: "BAZEL_SCRIPT"
   value: "tools/internal_ci/linux/grpc_bazel_build_in_docker.sh"

tools/internal_ci/linux/grpc_bazel_build_in_docker.sh

@@ -12,38 +12,34 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Test basic Bazel features
-#
-# NOTE: No empty lines should appear in this file before igncr is set!
-set -ex -o igncr || set -ex
-
-mkdir -p /var/local/git
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
-cd /var/local/git/grpc
 
 # Build all basic targets using the strict warning option which leverages the
 # clang compiler to check if sources can pass a set of warning options.
 # For now //examples/android/binder/ are excluded because it needs Android
 # SDK/NDK to be installed to build
-bazel build --define=use_strict_warning=true \
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_build_with_strict_warnings
-	-- \
+bazel_build_with_strict_warnings/bazel_wrapper \
-	:all \
+  --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \
-	//src/core/... \
+  build \
-	//src/compiler/... \
+  --define=use_strict_warning=true \
-	//test/... \
+  -- \
-	//examples/... \
+  :all \
-	-//examples/android/binder/...
+  //src/core/... \
+  //src/compiler/... \
+  //test/... \
+  //examples/... \
+  -//examples/android/binder/...
 
 # TODO(veblush): Remove this test after migration to abseil-status is done.
-bazel build --define=use_strict_warning=true --define=use_abseil_status=true \
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_build_with_abseil_status
-	-- \
+bazel_build_with_abseil_status/bazel_wrapper \
-	//src/core/... \
+  --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \
-	//src/compiler/... \
+  build \
-	//test/...
+  --define=use_strict_warning=true --define=use_abseil_status=true \
+  -- \
+  //src/core/... \
+  //src/compiler/... \
+  //test/...
 
 # TODO(jtattersmusch): Adding a build here for --define=grpc_no_xds is not ideal
 # and we should find a better place for this. Refer
@@ -56,6 +52,6 @@ EXIT_CODE=0
 bazel build //test/cpp/end2end/xds:xds_end2end_test --define=grpc_no_xds=true || EXIT_CODE=$?
 if [ $EXIT_CODE -eq 0 ]
 then
-	echo "Building xds_end2end_test succeeded even with --define=grpc_no_xds=true"
+  echo "Building xds_end2end_test succeeded even with --define=grpc_no_xds=true"
-	exit 1
+  exit 1
 fi

tools/internal_ci/linux/grpc_binder_transport_apk.cfg

@@ -17,6 +17,13 @@
 # Location of the continuous shell script in repository.
 build_file: "grpc/tools/internal_ci/linux/grpc_binder_transport_apk.sh"
 timeout_mins: 60
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
+}
+
 env_vars {
   key: "BAZEL_SCRIPT"
   value: "tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh"

tools/internal_ci/linux/grpc_binder_transport_apk.sh

@@ -15,11 +15,13 @@
 
 set -ex
 
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
 # change to grpc repo root
 cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 export DOCKERFILE_DIR=tools/dockerfile/test/binder_transport_apk
-export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT
+exec tools/run_tests/dockerize/build_and_run_docker.sh "${BAZEL_SCRIPT}"
-exec tools/run_tests/dockerize/build_and_run_docker.sh

tools/internal_ci/linux/grpc_binder_transport_apk_build_in_docker.sh

@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
 # Copyright 2021 gRPC authors.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,17 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
+set -ex
-#
-# NOTE: No empty lines should appear in this file before igncr is set!
-set -ex -o igncr || set -ex
-
-mkdir -p /var/local/git
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
-cd /var/local/git/grpc
 
 echo $ANDROID_HOME
 echo $ANDROID_NDK_HOME
@@ -32,7 +23,11 @@ echo $ANDROID_NDK_HOME
 # CPU are specified because gRPC does not build with 32bit NDK (which has socklen_t
 # defined as int due to an accident).
 # The python option is for disabling python2 enforcement when packing APK
-bazel build --define=use_strict_warning=true \
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_binder_example_app
+bazel_binder_example_app/bazel_wrapper \
+  --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \
+  build \
+  --define=use_strict_warning=true \
   --fat_apk_cpu=x86_64,arm64-v8a \
   --extra_toolchains=@rules_python//python:autodetecting_toolchain_nonstrict \
   //examples/android/binder/java/io/grpc/binder/cpp/exampleclient:app \
@@ -40,5 +35,9 @@ bazel build --define=use_strict_warning=true \
 
 # Make sure the Java code that will be invoked by binder transport
 # implementation builds
-bazel build --define=use_strict_warning=true \
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_binder_connection_helper
+bazel_binder_connection_helper/bazel_wrapper \
+  --bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc \
+  build \
+  --define=use_strict_warning=true \
   @binder_transport_android_helper//io/grpc/binder/cpp:connection_helper

tools/internal_ci/linux/grpc_feature_example_tests.cfg

@@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
 timeout_mins: 30
 env_vars {
   key: "BAZEL_SCRIPT"
-  value: "tools/internal_ci/linux/grpc_feature_example_tests.sh"
+  value: "tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh"
 }
 action {
   define_artifacts {

tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh

@@ -12,17 +12,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#!/usr/bin/env bash
-#
-# NOTE: No empty lines should appear in this file before igncr is set!
-set -ex -o igncr || set -ex
 
-mkdir -p /var/local/git
+set -ex
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
-cd /var/local/git/grpc
 
 apt-get install -y lsof
 

tools/internal_ci/linux/grpc_flaky_network.cfg

@@ -15,9 +15,11 @@
 # Config file for the internal CI (in protobuf text format)
 
 # Location of the continuous shell script in repository.
-build_file: "grpc/tools/internal_ci/linux/grpc_bazel_privileged_docker.sh"
+build_file: "grpc/tools/internal_ci/linux/grpc_flaky_network.sh"
 timeout_mins: 240
-env_vars {
+action {
-  key: "BAZEL_SCRIPT"
+  define_artifacts {
-  value: "tools/internal_ci/linux/grpc_flaky_network_in_docker.sh"
+    regex: "**/*sponge_log.*"
+    regex: "github/grpc/reports/**"
+  }
 }

tools/internal_ci/linux/grpc_flaky_network.sh

@@ -15,12 +15,15 @@
 
 set -ex
 
+# avoid slow finalization after the script has exited.
+source $(dirname $0)/../../../tools/internal_ci/helper_scripts/move_src_tree_and_respawn_itself_rc
+
 # change to grpc repo root
 cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
 export DOCKERFILE_DIR=tools/dockerfile/test/bazel
-export DOCKER_RUN_SCRIPT=$BAZEL_SCRIPT
 # NET_ADMIN capability allows tests to manipulate network interfaces
-exec tools/run_tests/dockerize/build_and_run_docker.sh --cap-add NET_ADMIN
+export EXTRA_DOCKER_ARGS="--cap-add NET_ADMIN"
+exec tools/run_tests/dockerize/build_and_run_docker.sh tools/internal_ci/linux/grpc_flaky_network_in_docker.sh

tools/internal_ci/linux/grpc_flaky_network_in_docker.sh

@@ -15,17 +15,10 @@
 #
 # Run the flaky network test
 #
-# NOTE: No empty lines should appear in this file before igncr is set!
+set -ex
-set -ex -o igncr || set -ex
-
-mkdir -p /var/local/git
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
-cd /var/local/git/grpc/test/cpp/end2end
 
 # iptables is used to drop traffic between client and server
 apt-get install -y iptables iproute2
 
-bazel test --test_output=all --test_timeout=1200 :flaky_network_test --test_env=GRPC_TRACE=http --test_env=GRPC_VERBOSITY=DEBUG
+python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path bazel_flaky_network_test
+bazel_flaky_network_test/bazel_wrapper test --test_output=all --test_timeout=1200 //test/cpp/end2end:flaky_network_test --test_env=GRPC_TRACE=http --test_env=GRPC_VERBOSITY=DEBUG

tools/internal_ci/linux/grpc_python_bazel_test_in_docker.sh

@@ -12,25 +12,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# Test full Bazel
 
 set -ex
 
-mkdir -p /var/local/git
+RESULTSTORE_RESULTS_FLAG="--bazelrc=tools/remote_build/include/test_locally_with_resultstore_results.bazelrc"
-git clone /var/local/jenkins/grpc /var/local/git/grpc
-(cd /var/local/jenkins/grpc/ && git submodule foreach 'cd /var/local/git/grpc \
-&& git submodule update --init --reference /var/local/jenkins/grpc/${name} \
-${name}')
-cd /var/local/git/grpc
 TEST_TARGETS="//src/python/... //tools/distrib/python/grpcio_tools/... //examples/python/..."
 BAZEL_FLAGS="--test_output=errors"
 
 python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests
-python_bazel_tests/bazel_wrapper test ${BAZEL_FLAGS} ${TEST_TARGETS}
+python_bazel_tests/bazel_wrapper ${RESULTSTORE_RESULTS_FLAG} test ${BAZEL_FLAGS} ${TEST_TARGETS}
 
 python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_single_threaded_unary_streams
-python_bazel_tests_single_threaded_unary_streams/bazel_wrapper test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS}
+python_bazel_tests_single_threaded_unary_streams/bazel_wrapper ${RESULTSTORE_RESULTS_FLAG} test --config=python_single_threaded_unary_stream ${BAZEL_FLAGS} ${TEST_TARGETS}
 
 python3 tools/run_tests/python_utils/bazel_report_helper.py --report_path python_bazel_tests_poller_engine
-python_bazel_tests_poller_engine/bazel_wrapper test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS}
+python_bazel_tests_poller_engine/bazel_wrapper ${RESULTSTORE_RESULTS_FLAG} test --config=python_poller_engine ${BAZEL_FLAGS} ${TEST_TARGETS}

tools/internal_ci/linux/pull_request/grpc_feature_example_tests.cfg

@@ -19,7 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_bazel.sh"
 timeout_mins: 30
 env_vars {
   key: "BAZEL_SCRIPT"
-  value: "tools/internal_ci/linux/grpc_feature_example_tests.sh"
+  value: "tools/internal_ci/linux/grpc_feature_example_tests_in_docker.sh"
 }
 action {
   define_artifacts {

tools/run_tests/dockerize/docker_propagate_env.list

@@ -12,6 +12,7 @@ KOKORO_BUILD_INITIATOR
 KOKORO_BUILD_NUMBER
 KOKORO_BUILD_URL
 KOKORO_GIT_COMMIT
+KOKORO_GITHUB_COMMIT
 KOKORO_JOB_NAME
 KOKORO_GITHUB_PULL_REQUEST_NUMBER
 KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH