Merge pull request #14691 from jtattermusch/kokoro_packages_distribtest

Add kokoro build_packages and distribtests jobs.
pull/14707/head
Jan Tattermusch 7 years ago committed by GitHub
commit 261448e357
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      src/csharp/build_packages_dotnetcli.bat
  2. 6
      src/csharp/build_packages_dotnetcli.sh
  3. 8
      templates/src/csharp/build_packages_dotnetcli.bat.template
  4. 6
      templates/src/csharp/build_packages_dotnetcli.sh.template
  5. 26
      tools/internal_ci/linux/grpc_build_packages.cfg
  6. 35
      tools/internal_ci/linux/grpc_build_packages.sh
  7. 26
      tools/internal_ci/linux/grpc_distribtests.cfg
  8. 35
      tools/internal_ci/linux/grpc_distribtests.sh
  9. 27
      tools/internal_ci/macos/grpc_distribtests.cfg
  10. 27
      tools/internal_ci/macos/grpc_distribtests.sh
  11. 35
      tools/internal_ci/windows/grpc_build_packages.bat
  12. 26
      tools/internal_ci/windows/grpc_build_packages.cfg
  13. 35
      tools/internal_ci/windows/grpc_distribtests.bat
  14. 26
      tools/internal_ci/windows/grpc_distribtests.cfg
  15. 7
      tools/run_tests/artifacts/build_package_php.sh
  16. 5
      tools/run_tests/artifacts/build_package_python.sh
  17. 12
      tools/run_tests/artifacts/build_package_ruby.sh
  18. 4
      tools/run_tests/dockerize/build_and_run_docker.sh

@ -19,17 +19,21 @@ set VERSION=1.11.0-dev
set NUGET=C:\nuget\nuget.exe set NUGET=C:\nuget\nuget.exe
set DOTNET=dotnet set DOTNET=dotnet
set -ex
mkdir ..\..\artifacts mkdir ..\..\artifacts
@rem Collect the artifacts built by the previous build step if running on Jenkins @rem Collect the artifacts built by the previous build step if running on Jenkins
mkdir nativelibs mkdir nativelibs
@rem Jenkins flow (deprecated)
powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs" powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs"
@rem Kokoro flow
powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs"
@rem Collect protoc artifacts built by the previous build step @rem Collect protoc artifacts built by the previous build step
mkdir protoc_plugins mkdir protoc_plugins
@rem Jenkins flow (deprecated)
powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins" powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins"
@rem Kokoro flow
powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins"
%DOTNET% restore Grpc.sln || goto :error %DOTNET% restore Grpc.sln || goto :error

@ -21,11 +21,17 @@ mkdir -p ../../artifacts/
# Collect the artifacts built by the previous build step # Collect the artifacts built by the previous build step
mkdir -p nativelibs mkdir -p nativelibs
# Jenkins flow (deprecated)
cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true
# Kokoro flow
cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true
# Collect protoc artifacts built by the previous build step # Collect protoc artifacts built by the previous build step
mkdir -p protoc_plugins mkdir -p protoc_plugins
# Jenkins flow (deprecated)
cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true
# Kokoro flow
cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true
dotnet restore Grpc.sln dotnet restore Grpc.sln

@ -21,17 +21,21 @@
set NUGET=C:\nuget\nuget.exe set NUGET=C:\nuget\nuget.exe
set DOTNET=dotnet set DOTNET=dotnet
set -ex
mkdir ..\..\artifacts mkdir ..\..\artifacts
@rem Collect the artifacts built by the previous build step if running on Jenkins @rem Collect the artifacts built by the previous build step if running on Jenkins
mkdir nativelibs mkdir nativelibs
@rem Jenkins flow (deprecated)
powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs" powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs"
@rem Kokoro flow
powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs"
@rem Collect protoc artifacts built by the previous build step @rem Collect protoc artifacts built by the previous build step
mkdir protoc_plugins mkdir protoc_plugins
@rem Jenkins flow (deprecated)
powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins" powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins"
@rem Kokoro flow
powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins"
%%DOTNET% restore Grpc.sln || goto :error %%DOTNET% restore Grpc.sln || goto :error

@ -23,11 +23,17 @@
# Collect the artifacts built by the previous build step # Collect the artifacts built by the previous build step
mkdir -p nativelibs mkdir -p nativelibs
# Jenkins flow (deprecated)
cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true
# Kokoro flow
cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true
# Collect protoc artifacts built by the previous build step # Collect protoc artifacts built by the previous build step
mkdir -p protoc_plugins mkdir -p protoc_plugins
# Jenkins flow (deprecated)
cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true
# Kokoro flow
cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true
dotnet restore Grpc.sln dotnet restore Grpc.sln

@ -0,0 +1,26 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/linux/grpc_build_packages.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/*sponge_log.xml"
regex: "github/grpc/reports/**"
regex: "github/grpc/artifacts/**"
}
}

@ -0,0 +1,35 @@
#!/bin/bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# change to grpc repo root
cd $(dirname $0)/../../..
source tools/internal_ci/helper_scripts/prepare_build_linux_rc
set +ex
[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
set -e # rvm commands are very verbose
rvm --default use ruby-2.4.1
set -ex
# Move artifacts generated by the previous step in the build chain to a place
# where they can be accessed from within a docker container that builds
# the packages
mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
ls -R input_artifacts || true
tools/run_tests/task_runner.py -f package linux -j 6

@ -0,0 +1,26 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/linux/grpc_distribtests.sh"
timeout_mins: 120
action {
define_artifacts {
regex: "**/*sponge_log.xml"
regex: "github/grpc/reports/**"
regex: "github/grpc/artifacts/**"
}
}

@ -0,0 +1,35 @@
#!/bin/bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# change to grpc repo root
cd $(dirname $0)/../../..
source tools/internal_ci/helper_scripts/prepare_build_linux_rc
set +ex
[[ -s /etc/profile.d/rvm.sh ]] && . /etc/profile.d/rvm.sh
set -e # rvm commands are very verbose
rvm --default use ruby-2.4.1
set -ex
# Move packages generated by the previous step in the build chain to a place
# where they can be accessed from within a docker container that run the
# distribtests
mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
ls -R input_artifacts || true
tools/run_tests/task_runner.py -f distribtest linux -j 6

@ -0,0 +1,27 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_distribtests.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {
define_artifacts {
regex: "**/*sponge_log.xml"
regex: "github/grpc/reports/**"
regex: "github/grpc/artifacts/**"
}
}

@ -0,0 +1,27 @@
#!/bin/bash
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
# change to grpc repo root
cd $(dirname $0)/../../..
source tools/internal_ci/helper_scripts/prepare_build_macos_rc
# Move packages generated by the previous step in the build chain.
mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true
ls -R input_artifacts || true
tools/run_tests/task_runner.py -f distribtest macos

@ -0,0 +1,35 @@
@rem Copyright 2017 gRPC authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
@rem TODO(jtattermusch): get rid of this hack
rename C:\Python27_32bit Python27_32bits
rename C:\Python34_32bit Python34_32bits
rename C:\Python35_32bit Python35_32bits
rename C:\Python36_32bit Python36_32bits
@rem enter repo root
cd /d %~dp0\..\..\..
call tools/internal_ci/helper_scripts/prepare_build_windows.bat
@rem Move artifacts generated by the previous step in the build chain.
powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
dir input_artifacts
python tools/run_tests/task_runner.py -f package windows -j 4 || goto :error
goto :EOF
:error
exit /b %errorlevel%

@ -0,0 +1,26 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/windows/grpc_build_packages.bat"
timeout_mins: 120
action {
define_artifacts {
regex: "**/*sponge_log.xml"
regex: "github/grpc/reports/**"
regex: "github/grpc/artifacts/**"
}
}

@ -0,0 +1,35 @@
@rem Copyright 2017 gRPC authors.
@rem
@rem Licensed under the Apache License, Version 2.0 (the "License");
@rem you may not use this file except in compliance with the License.
@rem You may obtain a copy of the License at
@rem
@rem http://www.apache.org/licenses/LICENSE-2.0
@rem
@rem Unless required by applicable law or agreed to in writing, software
@rem distributed under the License is distributed on an "AS IS" BASIS,
@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@rem See the License for the specific language governing permissions and
@rem limitations under the License.
@rem Move python installation from _32bit to _32bits where they are expected by python artifact builder
@rem TODO(jtattermusch): get rid of this hack
rename C:\Python27_32bit Python27_32bits
rename C:\Python34_32bit Python34_32bits
rename C:\Python35_32bit Python35_32bits
rename C:\Python36_32bit Python36_32bits
@rem enter repo root
cd /d %~dp0\..\..\..
call tools/internal_ci/helper_scripts/prepare_build_windows.bat
@rem Move packages generated by the previous step in the build chain.
powershell -Command "mv %KOKORO_GFILE_DIR%\github\grpc\artifacts input_artifacts"
dir input_artifacts
python tools/run_tests/task_runner.py -f distribtest windows -j 4 || goto :error
goto :EOF
:error
exit /b %errorlevel%

@ -0,0 +1,26 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/windows/grpc_distribtests.bat"
timeout_mins: 120
action {
define_artifacts {
regex: "**/*sponge_log.xml"
regex: "github/grpc/reports/**"
regex: "github/grpc/artifacts/**"
}
}

@ -17,5 +17,10 @@ set -ex
cd "$(dirname "$0")/../../.." cd "$(dirname "$0")/../../.."
# All the PHP packages have been built in the artifact phase already
# and we only collect them here to deliver them to the distribtest phase.
mkdir -p artifacts/ mkdir -p artifacts/
cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true # Jenkins flow (deprecated)
cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true
# Kokoro flow
cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true

@ -21,7 +21,10 @@ mkdir -p artifacts/
# All the python packages have been built in the artifact phase already # All the python packages have been built in the artifact phase already
# and we only collect them here to deliver them to the distribtest phase. # and we only collect them here to deliver them to the distribtest phase.
cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true # Jenkins flow (deprecated)
cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true
# Kokoro flow
cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true
# TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz
# source distribution package, and only one of them will end up # source distribution package, and only one of them will end up

@ -23,7 +23,10 @@ mkdir -p artifacts/
# All the ruby packages have been built in the artifact phase already # All the ruby packages have been built in the artifact phase already
# and we only collect them here to deliver them to the distribtest phase. # and we only collect them here to deliver them to the distribtest phase.
cp -r "$EXTERNAL_GIT_ROOT"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true # Jenkins flow (deprecated)
cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true
# Kokoro flow
cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/ruby_native_gem_*/* artifacts/ || true
well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers )
@ -41,7 +44,12 @@ for arch in {x86,x64}; do
;; ;;
esac esac
for plat in {windows,linux,macos}; do for plat in {windows,linux,macos}; do
input_dir="$EXTERNAL_GIT_ROOT/platform=${plat}/artifacts/protoc_${plat}_${arch}" if [ "${KOKORO_JOB_NAME}" != "" ]
then
input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}"
else
input_dir="${EXTERNAL_GIT_ROOT}/platform=${plat}/artifacts/protoc_${plat}_${arch}"
fi
output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}"
mkdir -p "$output_dir"/google/protobuf mkdir -p "$output_dir"/google/protobuf
mkdir -p "$output_dir"/google/protobuf/compiler # needed for plugin.proto mkdir -p "$output_dir"/google/protobuf/compiler # needed for plugin.proto

@ -58,6 +58,10 @@ docker run \
"$@" \ "$@" \
-e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \ -e EXTERNAL_GIT_ROOT="/var/local/jenkins/grpc" \
-e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \ -e THIS_IS_REALLY_NEEDED='see https://github.com/docker/docker/issues/14203 for why docker is awful' \
-e "KOKORO_BUILD_ID=$KOKORO_BUILD_ID" \
-e "KOKORO_BUILD_NUMBER=$KOKORO_BUILD_NUMBER" \
-e "KOKORO_BUILD_URL=$KOKORO_BUILD_URL" \
-e "KOKORO_JOB_NAME=$KOKORO_JOB_NAME" \
-v "$git_root:/var/local/jenkins/grpc:ro" \ -v "$git_root:/var/local/jenkins/grpc:ro" \
-w /var/local/git/grpc \ -w /var/local/git/grpc \
--name="$CONTAINER_NAME" \ --name="$CONTAINER_NAME" \

Loading…
Cancel
Save