Merge pull request #21412 from jtattermusch/leaner_builds

Leaner MacOS tests (selective installation of prerequisites).
pull/21475/head
Jan Tattermusch 5 years ago committed by GitHub
commit 56f15492ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 134
      tools/internal_ci/helper_scripts/prepare_build_macos_rc
  2. 2
      tools/internal_ci/macos/grpc_basictests_cpp_ios.cfg
  3. 2
      tools/internal_ci/macos/grpc_basictests_csharp.cfg
  4. 19
      tools/internal_ci/macos/grpc_basictests_csharp.sh
  5. 31
      tools/internal_ci/macos/grpc_basictests_dbg.cfg
  6. 2
      tools/internal_ci/macos/grpc_basictests_objc_examples.cfg
  7. 2
      tools/internal_ci/macos/grpc_basictests_objc_ios.cfg
  8. 2
      tools/internal_ci/macos/grpc_basictests_objc_mac.cfg
  9. 2
      tools/internal_ci/macos/grpc_basictests_objc_tvos.cfg
  10. 31
      tools/internal_ci/macos/grpc_basictests_opt.cfg
  11. 2
      tools/internal_ci/macos/grpc_basictests_python.cfg
  12. 19
      tools/internal_ci/macos/grpc_basictests_python.sh
  13. 2
      tools/internal_ci/macos/grpc_basictests_ruby.cfg
  14. 19
      tools/internal_ci/macos/grpc_basictests_ruby.sh
  15. 3
      tools/internal_ci/macos/grpc_build_artifacts.sh
  16. 1
      tools/internal_ci/macos/grpc_distribtests.sh
  17. 1
      tools/internal_ci/macos/grpc_ios_binary_size.sh
  18. 19
      tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh
  19. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_cpp_ios.cfg
  20. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_csharp.cfg
  21. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_objc_examples.cfg
  22. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_objc_ios.cfg
  23. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_objc_mac.cfg
  24. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_objc_tvos.cfg
  25. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_python.cfg
  26. 2
      tools/internal_ci/macos/pull_request/grpc_basictests_ruby.cfg

@ -18,6 +18,13 @@
launchctl limit maxfiles
ulimit -a
# WARNING: TRY TO KEEP THIS FILE AS LEAN AS POSSIBLE
# The rules:
# - think twice before adding installer that takes long time (builds are already slow)
# - do not add any steps that are flaky (builds will become flaky)
# - try adding a dependency under a language-specific section first (reduces latency and increases build stability)
# - only add stuff that you absolutely need for your builds to work (add comment to explain why its needed)
# Add GCP credentials for BQ access
pip install --user google-api-python-client oauth2client
export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json
@ -27,62 +34,85 @@ if [ -n "$KOKORO_GITHUB_PULL_REQUEST_NUMBER" ]; then
export RUN_TESTS_FLAGS="--filter_pr_tests --base_branch origin/$KOKORO_GITHUB_PULL_REQUEST_TARGET_BRANCH $RUN_TESTS_FLAGS"
fi
set +ex # rvm script is very verbose and exits with errorcode
# Advice from https://github.com/Homebrew/homebrew-cask/issues/8629#issuecomment-68641176
brew update && brew upgrade brew-cask && brew cleanup && brew cask cleanup
rvm --debug requirements ruby-2.5.0
source $HOME/.rvm/scripts/rvm
set -e # rvm commands are very verbose
time rvm install 2.5.0
rvm use 2.5.0 --default
time gem install bundler -v 1.17.3 --no-ri --no-doc
time gem install cocoapods --version 1.3.1 --no-ri --no-doc
time gem install rake-compiler --no-ri --no-doc
rvm osx-ssl-certs status all
rvm osx-ssl-certs update all
set -ex
# cocoapods
export LANG=en_US.UTF-8
# pre-fetch cocoapods master repo's most recent commit only
mkdir -p ~/.cocoapods/repos
time git clone --depth 1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master
# python
time pip install --user virtualenv
time pip install --user --upgrade Mako six tox setuptools twisted pyyaml pyjwt cryptography requests
export PYTHONPATH=/Library/Python/3.4/site-packages
# Install Python 3.7 and Python 3.8
time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg
echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > /tmp/python_installer_checksum.md5
echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5
md5sum -c /tmp/python_installer_checksum.md5
time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target /
# set xcode version for Obj-C tests
sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/
# Disable some unwanted dotnet options
export NUGET_XMLDOC_MODE=skip
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true
export DOTNET_CLI_TELEMETRY_OPTOUT=true
if [ "${PREPARE_BUILD_INSTALL_DEPS_RUBY}" == "true" ]
then
set +ex # rvm script is very verbose and exits with errorcode
# Advice from https://github.com/Homebrew/homebrew-cask/issues/8629#issuecomment-68641176
brew update && brew upgrade brew-cask && brew cleanup && brew cask cleanup
rvm --debug requirements ruby-2.5.0
source $HOME/.rvm/scripts/rvm
set -e # rvm commands are very verbose
time rvm install 2.5.0
rvm use 2.5.0 --default
time gem install bundler -v 1.17.3 --no-ri --no-doc
time gem install cocoapods --version 1.3.1 --no-ri --no-doc
time gem install rake-compiler --no-ri --no-doc
rvm osx-ssl-certs status all
rvm osx-ssl-certs update all
set -ex
fi
# TODO(jtattermusch): better debugging of clock skew, remove once not needed
date
if [ "${PREPARE_BUILD_INSTALL_DEPS_OBJC}" == "true" ]
then
# cocoapods
export LANG=en_US.UTF-8
# pre-fetch cocoapods master repo's most recent commit only
mkdir -p ~/.cocoapods/repos
time git clone --depth 1 https://github.com/CocoaPods/Specs.git ~/.cocoapods/repos/master
time git submodule update --init
# set xcode version for Obj-C tests
sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/
# Needed for ios-binary-size
time pip install --user pyyaml pyjwt cryptography requests
# Store intermediate build files of ObjC tests into /tmpfs
# TODO(jtattermusch): this has likely been done to avoid running
# out of disk space when running ios-binary-size tests, but
# it would be good to revisit.
mkdir /tmpfs/Build-ios-binary-size
ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build
mkdir -p /tmpfs/DerivedData
rm -rf ~/Library/Developer/Xcode/DerivedData
mkdir -p ~/Library/Developer/Xcode
ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData
fi
if [ "${PREPARE_BUILD_INSTALL_DEPS_PYTHON}" == "true" ]
then
# python
time pip install --user virtualenv
time pip install --user --upgrade Mako six tox setuptools twisted pyyaml pyjwt cryptography requests
export PYTHONPATH=/Library/Python/3.4/site-packages
# Store intermediate build files of ObjC tests into /tmpfs
mkdir /tmpfs/Build-ios-binary-size
ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build
mkdir -p /tmpfs/DerivedData
rm -rf ~/Library/Developer/Xcode/DerivedData
mkdir -p ~/Library/Developer/Xcode
ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData
# make sure md5sum is available (requires coreutils 8.31+)
brew upgrade coreutils
# Install Python 3.7 and Python 3.8
time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg
time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg
echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > /tmp/python_installer_checksum.md5
echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5
md5sum -c /tmp/python_installer_checksum.md5
time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target /
time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target /
fi
if [ "${PREPARE_BUILD_INSTALL_DEPS_CSHARP}" == "true" ]
then
# Disable some unwanted dotnet options
export NUGET_XMLDOC_MODE=skip
export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true
export DOTNET_CLI_TELEMETRY_OPTOUT=true
# Xcode 10+ doesn't support building for i386 architecture
sudo xcode-select -switch /Applications/Xcode_9.2.app/Contents/Developer/
fi
# PHP tests currently require using an older version of PHPUnit
ln -sf /usr/local/bin/phpunit-5.7 /usr/local/bin/phpunit
# TODO(jtattermusch): better debugging of clock skew, remove once not needed
date
time git submodule update --init

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_basictests_csharp.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 60
action {

@ -0,0 +1,19 @@
#!/bin/bash
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export PREPARE_BUILD_INSTALL_DEPS_CSHARP=true
$(dirname $0)/grpc_run_tests_matrix.sh

@ -1,31 +0,0 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 240
action {
define_artifacts {
regex: "**/*sponge_log.*"
regex: "github/grpc/reports/**"
}
}
env_vars {
key: "RUN_TESTS_FLAGS"
value: "-f basictests macos dbg --exclude objc --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results"
}

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -1,31 +0,0 @@
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 240
action {
define_artifacts {
regex: "**/*sponge_log.*"
regex: "github/grpc/reports/**"
}
}
env_vars {
key: "RUN_TESTS_FLAGS"
value: "-f basictests macos opt --exclude objc --internal_ci -j 1 --inner_jobs 4 --bq_result_table aggregate_results"
}

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_basictests_python.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 60
action {

@ -0,0 +1,19 @@
#!/bin/bash
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export PREPARE_BUILD_INSTALL_DEPS_PYTHON=true
$(dirname $0)/grpc_run_tests_matrix.sh

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_basictests_ruby.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 60
action {

@ -0,0 +1,19 @@
#!/bin/bash
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export PREPARE_BUILD_INSTALL_DEPS_RUBY=true
$(dirname $0)/grpc_run_tests_matrix.sh

@ -18,6 +18,9 @@ set -ex
# change to grpc repo root
cd $(dirname $0)/../../..
export PREPARE_BUILD_INSTALL_DEPS_CSHARP=true
export PREPARE_BUILD_INSTALL_DEPS_PYTHON=true
export PREPARE_BUILD_INSTALL_DEPS_RUBY=true
source tools/internal_ci/helper_scripts/prepare_build_macos_rc
# install cython for all python versions

@ -18,6 +18,7 @@ set -ex
# change to grpc repo root
cd $(dirname $0)/../../..
export PREPARE_BUILD_INSTALL_DEPS_CSHARP=true
source tools/internal_ci/helper_scripts/prepare_build_macos_rc
# Move packages generated by the previous step in the build chain.

@ -21,6 +21,7 @@ set -ex
# Enter the gRPC repo root
cd $(dirname $0)/../../..
export PREPARE_BUILD_INSTALL_DEPS_OBJC=true
source tools/internal_ci/helper_scripts/prepare_build_macos_rc
tools/profiling/ios_bin/binary_size.py \

@ -0,0 +1,19 @@
#!/bin/bash
# Copyright 2019 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -ex
export PREPARE_BUILD_INSTALL_DEPS_OBJC=true
$(dirname $0)/grpc_run_tests_matrix.sh

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_basictests_csharp.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 60
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix_objc.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 120
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_basictests_python.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 60
action {

@ -15,7 +15,7 @@
# Config file for the internal CI (in protobuf text format)
# Location of the continuous shell script in repository.
build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh"
build_file: "grpc/tools/internal_ci/macos/grpc_basictests_ruby.sh"
gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json"
timeout_mins: 60
action {

Loading…
Cancel
Save