Merge pull request #14849 from mehrdada/v1.10.x

Backport #14691 (Kokoro package build and distrib test config) to v1.10.x

src/csharp/build_packages_dotnetcli.bat

@@ -19,17 +19,21 @@ set VERSION=1.10.1-pre1
 set NUGET=C:\nuget\nuget.exe
 set DOTNET=dotnet
 
-set -ex
-
 mkdir ..\..\artifacts
 
 @rem Collect the artifacts built by the previous build step if running on Jenkins
 mkdir nativelibs
+@rem Jenkins flow (deprecated)
 powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs"
+@rem Kokoro flow
+powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs"
 
 @rem Collect protoc artifacts built by the previous build step
 mkdir protoc_plugins
+@rem Jenkins flow (deprecated)
 powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins"
+@rem Kokoro flow
+powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins"
 
 %DOTNET% restore Grpc.sln || goto :error
 

src/csharp/build_packages_dotnetcli.sh

@@ -21,11 +21,17 @@ mkdir -p ../../artifacts/
 
 # Collect the artifacts built by the previous build step
 mkdir -p nativelibs
+# Jenkins flow (deprecated)
 cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true
+# Kokoro flow
+cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true
 
 # Collect protoc artifacts built by the previous build step
 mkdir -p protoc_plugins
+# Jenkins flow (deprecated)
 cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true
+# Kokoro flow
+cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true
 
 dotnet restore Grpc.sln
 

templates/src/csharp/build_packages_dotnetcli.bat.template

@@ -21,17 +21,21 @@
   set NUGET=C:\nuget\nuget.exe
   set DOTNET=dotnet
   
-  set -ex
-  
   mkdir ..\..\artifacts
   
   @rem Collect the artifacts built by the previous build step if running on Jenkins
   mkdir nativelibs
+  @rem Jenkins flow (deprecated)
   powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs"
+  @rem Kokoro flow
+  powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs"
   
   @rem Collect protoc artifacts built by the previous build step
   mkdir protoc_plugins
+  @rem Jenkins flow (deprecated)
   powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins"
+  @rem Kokoro flow
+  powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins"
   
   %%DOTNET% restore Grpc.sln || goto :error
   

templates/src/csharp/build_packages_dotnetcli.sh.template

@@ -23,11 +23,17 @@
   
   # Collect the artifacts built by the previous build step
   mkdir -p nativelibs
+  # Jenkins flow (deprecated)
   cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true
+  # Kokoro flow
+  cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true
   
   # Collect protoc artifacts built by the previous build step
   mkdir -p protoc_plugins
+  # Jenkins flow (deprecated)
   cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true
+  # Kokoro flow
+  cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true
   
   dotnet restore Grpc.sln
   

tools/internal_ci/linux/grpc_build_packages.cfg

@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/linux/grpc_build_packages.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Move artifacts generated by the previous step in the build chain to a place
+# where they can be accessed from within a docker container that builds
+# the packages
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f package linux -j 6

tools/internal_ci/linux/grpc_distribtests.cfg

@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/linux/grpc_distribtests.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_linux_rc
+
+set +ex
+[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
+set -e  # rvm commands are very verbose
+rvm --default use ruby-2.4.1
+set -ex
+
+# Move packages generated by the previous step in the build chain to a place
+# where they can be accessed from within a docker container that run the
+# distribtests
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f distribtest linux -j 6

tools/internal_ci/macos/grpc_distribtests.cfg

@@ -0,0 +1,27 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/macos/grpc_distribtests.sh"
+gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/macos/grpc_distribtests.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+# change to grpc repo root
+cd $(dirname $0)/../../..
+
+source tools/internal_ci/helper_scripts/prepare_build_macos_rc
+
+# Move packages generated by the previous step in the build chain.
+mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
+ls -R input_artifacts || true
+
+tools/run_tests/task_runner.py -f distribtest macos

tools/internal_ci/windows/grpc_build_packages.bat

@@ -0,0 +1,35 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+@rem Move artifacts generated by the previous step in the build chain.
+powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
+dir input_artifacts
+
+python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%

tools/internal_ci/windows/grpc_build_packages.cfg

@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/internal_ci/windows/grpc_distribtests.bat

@@ -0,0 +1,35 @@
+@rem Copyright 2017 gRPC authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem     http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+
+@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
+@rem TODO(jtattermusch): get rid of this hack
+rename C:\Python27_32bit Python27_32bits
+rename C:\Python34_32bit Python34_32bits
+rename C:\Python35_32bit Python35_32bits
+rename C:\Python36_32bit Python36_32bits
+
+@rem enter repo root
+cd /d %~dp0\..\..\..
+
+call tools/internal_ci/helper_scripts/prepare_build_windows.bat
+
+@rem Move packages generated by the previous step in the build chain.
+powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
+dir input_artifacts
+
+python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error
+goto :EOF
+
+:error
+exit /b %errorlevel%

tools/internal_ci/windows/grpc_distribtests.cfg

@@ -0,0 +1,26 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Config file for the internal CI (in protobuf text format)
+
+# Location of the continuous shell script in repository.
+build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat"
+timeout_mins: 120
+action {
+  define_artifacts {
+    regex: "**/*sponge_log.xml"
+    regex: "github/grpc/reports/**"
+    regex: "github/grpc/artifacts/**"
+  }
+}

tools/run_tests/artifacts/build_package_php.sh

@@ -17,5 +17,10 @@ set -ex
 
 cd "$(dirname "$0")/../../.."
 
+# All the PHP packages have been built in the artifact phase already
+# and we only collect them here to deliver them to the distribtest phase.
 mkdir -p artifacts/
-cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true
+# Jenkins flow (deprecated)
+cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true
+# Kokoro flow
+cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true

tools/run_tests/artifacts/build_package_python.sh

@@ -21,7 +21,10 @@ mkdir -p artifacts/
 
 # All the python packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
-cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true
+# Jenkins flow (deprecated)
+cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true
+# Kokoro flow
+cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true
 
 # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz
 # source distribution package, and only one of them will end up

tools/run_tests/artifacts/build_package_ruby.sh

@@ -23,7 +23,10 @@ mkdir -p artifacts/
 
 # All the ruby packages have been built in the artifact phase already
 # and we only collect them here to deliver them to the distribtest phase.
-cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true
+# Jenkins flow (deprecated)
+cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true
+# Kokoro flow
+cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/ruby_native_gem_*/* artifacts/ || true
 
 well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
 
@@ -41,7 +44,12 @@ for arch in {x86,x64}; do
       ;;
   esac
   for plat in {windows,linux,macos}; do
-    input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}"
+    if [ "${KOKORO_JOB_NAME}" != "" ]
+    then
+      input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}"
+    else
+      input_dir="${EXTERNAL_GIT_ROOT}/platform=${plat}/artifacts/protoc_${plat}_${arch}"
+    fi
     output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}"
     mkdir -p "$output_dir"/google/protobuf
     mkdir -p "$output_dir"/google/protobuf/compiler  # needed for plugin.proto

tools/run_tests/dockerize/build_and_run_docker.sh

@@ -58,6 +58,10 @@ docker run \
   "$@" \
   -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \
   -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
+  -e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \
+  -e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \
+  -e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \
+  -e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \
   -v "$git_root:/var/local/jenkins/grpc:ro" \
   -w /var/local/git/grpc \
   --name="$CONTAINER_NAME" \