add task_runner.py --inner_jobs support for C++ distribtests (#28594)

* add support for task_runner.py --inner_jobs for C++ distribtests

* fine tune linux c++ distribtests parallelism

* yapf code

test/distrib/cpp/run_distrib_test_cmake.bat

@@ -29,6 +29,8 @@ powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [Sys
 @rem set absolute path to OpenSSL with forward slashes
 set OPENSSL_DIR=%cd:\=/%/OpenSSL-Win32
 
+@rem TODO(jtattermusch): add support for GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS env variable
+
 @rem Install absl
 mkdir third_party\abseil-cpp\cmake\build
 pushd third_party\abseil-cpp\cmake\build

test/distrib/cpp/run_distrib_test_cmake.sh

@@ -20,11 +20,14 @@ cd "$(dirname "$0")/../../.."
 # Install openssl (to use instead of boringssl)
 apt-get update && apt-get install -y libssl-dev
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Install absl
 mkdir -p "third_party/abseil-cpp/cmake/build"
 pushd "third_party/abseil-cpp/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install c-ares
@@ -34,28 +37,28 @@ popd
 mkdir -p "third_party/cares/cares/cmake/build"
 pushd "third_party/cares/cares/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install protobuf
 mkdir -p "third_party/protobuf/cmake/build"
 pushd "third_party/protobuf/cmake/build"
 cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release ..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install re2
 mkdir -p "third_party/re2/cmake/build"
 pushd "third_party/re2/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install zlib
 mkdir -p "third_party/zlib/cmake/build"
 pushd "third_party/zlib/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Just before installing gRPC, wipe out contents of all the submodules to simulate
@@ -77,12 +80,12 @@ cmake \
   -DgRPC_SSL_PROVIDER=package \
   -DgRPC_ZLIB_PROVIDER=package \
   ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Build helloworld example using cmake
 mkdir -p "examples/cpp/helloworld/cmake/build"
 pushd "examples/cpp/helloworld/cmake/build"
 cmake ../..
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"
 popd

test/distrib/cpp/run_distrib_test_cmake_aarch64_cross.sh

@@ -26,6 +26,9 @@ wget -q -O cmake-linux.sh https://github.com/Kitware/CMake/releases/download/v3.
 sh cmake-linux.sh -- --skip-license --prefix=/usr
 rm cmake-linux.sh
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Build and install gRPC for the host architecture.
 # We do this because we need to be able to run protoc and grpc_cpp_plugin
 # while cross-compiling.
@@ -37,7 +40,7 @@ cmake \
   -DgRPC_BUILD_TESTS=OFF \
   -DgRPC_SSL_PROVIDER=package \
   ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Write a toolchain file to use for cross-compiling.
@@ -63,7 +66,7 @@ cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/toolchain.cmake \
       -DCMAKE_BUILD_TYPE=Release \
       -DCMAKE_INSTALL_PREFIX=/tmp/install \
       ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Build helloworld example for ARM.
@@ -77,5 +80,5 @@ cmake -DCMAKE_TOOLCHAIN_FILE=/tmp/toolchain.cmake \
       -DProtobuf_DIR=/tmp/stage/lib/cmake/protobuf \
       -DgRPC_DIR=/tmp/stage/lib/cmake/grpc \
       ../..
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"
 popd

test/distrib/cpp/run_distrib_test_cmake_as_externalproject.bat

@@ -26,6 +26,8 @@ powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [Sys
 @rem set absolute path to OpenSSL with forward slashes
 set OPENSSL_DIR=%cd:\=/%/OpenSSL-Win32
 
+@rem TODO(jtattermusch): add support for GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS env variable
+
 @rem Build helloworld example using cmake
 @rem Use non-standard build directory to avoid too long filenames
 mkdir example_build

test/distrib/cpp/run_distrib_test_cmake_as_externalproject.sh

@@ -28,9 +28,12 @@ rm -r third_party/bloaty/* || true
 rm -r third_party/boringssl-with-bazel/* || true
 rm -r third_party/googletest/* || true
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Build helloworld example using cmake superbuild
 cd examples/cpp/helloworld/cmake_externalproject
 mkdir -p cmake/build
 cd cmake/build
 cmake ../..
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"

test/distrib/cpp/run_distrib_test_cmake_as_submodule.sh

@@ -17,9 +17,12 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Build helloworld example using cmake, including grpc with "add_subdirectory"
 cd examples/cpp/helloworld
 mkdir -p cmake/build
 cd cmake/build
 cmake -DGRPC_AS_SUBMODULE=ON ../..
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"

test/distrib/cpp/run_distrib_test_cmake_fetchcontent.sh

@@ -27,6 +27,9 @@ wget -q -O cmake-linux.sh https://github.com/Kitware/CMake/releases/download/v3.
 sh cmake-linux.sh -- --skip-license --prefix=/usr
 rm cmake-linux.sh
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Build helloworld example.
 # This uses CMake's FetchContent module to download gRPC and its dependencies
 # and add it to the helloworld project as a subdirectory.
@@ -41,5 +44,5 @@ cmake \
   -DGRPC_FETCHCONTENT=ON \
   -DFETCHCONTENT_SOURCE_DIR_GRPC="$grpc_dir" \
   ../..
-make -j4
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"
 popd

test/distrib/cpp/run_distrib_test_cmake_module_install.sh

@@ -26,6 +26,9 @@ wget -q -O cmake-linux.sh https://github.com/Kitware/CMake/releases/download/v3.
 sh cmake-linux.sh -- --skip-license --prefix=/usr
 rm cmake-linux.sh
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Install gRPC and its dependencies
 mkdir -p "cmake/build"
 pushd "cmake/build"
@@ -35,12 +38,12 @@ cmake \
   -DgRPC_BUILD_TESTS=OFF \
   -DgRPC_SSL_PROVIDER=package \
   ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Build helloworld example using cmake
 mkdir -p "examples/cpp/helloworld/cmake/build"
 pushd "examples/cpp/helloworld/cmake/build"
 cmake ../..
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"
 popd

test/distrib/cpp/run_distrib_test_cmake_module_install_pkgconfig.sh

@@ -26,6 +26,9 @@ wget -q -O cmake-linux.sh https://github.com/Kitware/CMake/releases/download/v3.
 sh cmake-linux.sh -- --skip-license --prefix=/usr
 rm cmake-linux.sh
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Install gRPC and its dependencies
 mkdir -p "cmake/build"
 pushd "cmake/build"
@@ -35,11 +38,11 @@ cmake \
   -DgRPC_BUILD_TESTS=OFF \
   -DgRPC_SSL_PROVIDER=package \
   ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Build helloworld example using make & pkg-config
 cd examples/cpp/helloworld
 export PKG_CONFIG_PATH=/usr/local/grpc/lib/pkgconfig
 export PATH=$PATH:/usr/local/grpc/bin
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"

test/distrib/cpp/run_distrib_test_cmake_pkgconfig.sh

@@ -20,39 +20,42 @@ cd "$(dirname "$0")/../../.."
 # Install openssl (to use instead of boringssl)
 apt-get update && apt-get install -y libssl-dev
 
+# Use externally provided env to determine build parallelism, otherwise use default.
+GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS=${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS:-4}
+
 # Install absl
 mkdir -p "third_party/abseil-cpp/cmake/build"
 pushd "third_party/abseil-cpp/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install c-ares
 mkdir -p "third_party/cares/cares/cmake/build"
 pushd "third_party/cares/cares/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install protobuf
 mkdir -p "third_party/protobuf/cmake/build"
 pushd "third_party/protobuf/cmake/build"
 cmake -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release ..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install re2
 mkdir -p "third_party/re2/cmake/build"
 pushd "third_party/re2/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Install zlib
 mkdir -p "third_party/zlib/cmake/build"
 pushd "third_party/zlib/cmake/build"
 cmake -DCMAKE_BUILD_TYPE=Release ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Just before installing gRPC, wipe out contents of all the submodules to simulate
@@ -75,19 +78,19 @@ cmake \
   -DgRPC_SSL_PROVIDER=package \
   -DgRPC_ZLIB_PROVIDER=package \
   ../..
-make -j4 install
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}" install
 popd
 
 # Build helloworld example using Makefile and pkg-config
 pushd examples/cpp/helloworld
 export PKG_CONFIG_PATH=/usr/local/grpc/lib/pkgconfig
 export PATH=$PATH:/usr/local/grpc/bin
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"
 popd
 
 # Build route_guide example using Makefile and pkg-config
 pushd examples/cpp/route_guide
 export PKG_CONFIG_PATH=/usr/local/grpc/lib/pkgconfig
 export PATH=$PATH:/usr/local/grpc/bin
-make
+make "-j${GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS}"
 popd

tools/internal_ci/linux/grpc_distribtests.sh

@@ -41,4 +41,4 @@ set -ex
 mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
 ls -R input_artifacts || true
 
-tools/run_tests/task_runner.py -f distribtest linux -j 12
+tools/run_tests/task_runner.py -f distribtest linux -j 6 --inner_jobs 6

tools/internal_ci/linux/grpc_distribtests_standalone.sh

@@ -23,4 +23,4 @@ cd $(dirname $0)/../../..
 
 source tools/internal_ci/helper_scripts/prepare_build_linux_rc
 
-tools/run_tests/task_runner.py -f distribtest linux cpp ${TASK_RUNNER_EXTRA_FILTERS} -j 6
+tools/run_tests/task_runner.py -f distribtest linux cpp ${TASK_RUNNER_EXTRA_FILTERS} -j 6 --inner_jobs 6

tools/run_tests/artifacts/distribtest_targets.py

@@ -322,8 +322,12 @@ class CppDistribTest(object):
         return []
 
     def build_jobspec(self, inner_jobs=None):
-        # TODO(jtattermusch): honor inner_jobs arg for this task.
-        del inner_jobs
+        environ = {}
+        if inner_jobs is not None:
+            # set number of parallel jobs for the C++ build
+            environ['GRPC_CPP_DISTRIBTEST_BUILD_COMPILER_JOBS'] = str(
+                inner_jobs)
+
         if self.platform == 'linux':
             return create_docker_jobspec(
                 self.name,