From 76afc77ff62378755ab9b09a8720df99ed82b802 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 30 Nov 2020 14:13:00 -0800 Subject: [PATCH 1/4] Update PHP & Ruby Mac BUILD --- .../helper_scripts/prepare_build_macos_rc | 54 +++++-------------- .../internal_ci/macos/grpc_basictests_php.cfg | 2 +- .../internal_ci/macos/grpc_basictests_php.sh | 19 +++++++ .../internal_ci/macos/grpc_build_artifacts.sh | 1 + tools/run_tests/artifacts/artifact_targets.py | 2 +- 5 files changed, 35 insertions(+), 43 deletions(-) create mode 100644 tools/internal_ci/macos/grpc_basictests_php.sh diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index a8851ccc655..7f2ce40984e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -25,6 +25,9 @@ ulimit -a # - try adding a dependency under a language-specific section first (reduces latency and increases build stability) # - only add stuff that you absolutely need for your builds to work (add comment to explain why its needed) +# Disable HOMEBREW update to avoid new updates which potentially have problems. +export HOMEBREW_NO_AUTO_UPDATE=1 + # Add GCP credentials for BQ access pip install --user google-api-python-client oauth2client six==1.15.0 export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json @@ -36,34 +39,8 @@ fi if [ "${PREPARE_BUILD_INSTALL_DEPS_RUBY}" == "true" ] then - brew update - - # The latest gnupg version removes support for macos high sierra which would break ruby installation, - # so we need to pin a version before that. Unfortunately Homebrew is one of the most pathetic package - # managers out there and there is simply no way to install a specific version of gnupg. - # Instead, we force homebrew to use a slightly old version of the homebrew-core - # formula repository (before things got broken for us), so the homebrew formulas installed later - # will still work with MacOS high-sierra. - # Also https://github.com/Homebrew/homebrew-core/blob/07f2f9aab198ce369e24621b7c7224f63ffd27fb/Formula/gnupg.rb - # TODO(jtattermusch): migrate to MacOS mojave as soon as possible to avoid this ugly hack. - (cd /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core; git checkout 07f2f9aab198ce369e24621b7c7224f63ffd27fb) - # disable automatic brew update on "brew install" (which would ruin our explicit checkout) - export HOMEBREW_NO_AUTO_UPDATE=1 - # special case fix for https://github.com/grpc/grpc/issues/23027 - rm -f /usr/local/bin/gpg - rm -f /usr/local/bin/gpgconf - rm -f /usr/local/bin/gpgsm - # end https://github.com/grpc/grpc/issues/23027 - brew cleanup - - set +ex source $HOME/.rvm/scripts/rvm - set -ex for RUBY_VERSION in 2.5.0 2.7.0; do - # TODO(jtattermusch): find a better way of installing ruby, as the current way installs a huge number - # of completely unnecessary brew packages which 1. takes long time 2. is very prone to errors - # 3. generates a ton of logs making it super hard to debug when it breaks. - rvm --debug requirements "ruby-${RUBY_VERSION}" time rvm install "$RUBY_VERSION" time gem install bundler -v 1.17.3 --no-document time gem install rake-compiler --no-document @@ -104,20 +81,8 @@ then time pip install --user virtualenv time pip install --user --upgrade Mako tox setuptools==44.1.1 twisted pyyaml pyjwt cryptography requests - # make sure md5sum is available (requires coreutils 8.31+) - brew update-reset - brew upgrade coreutils - - # Install Python 3.7, 3.8, 3.9 - time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg - time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg + # Install Python 3.9 time curl -O https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg - echo "ae0717a02efea3b0eb34aadc680dc498 python-3.7.0-macosx10.9.pkg" > /tmp/python_installer_checksum.md5 - echo "f5f9ae9f416170c6355cab7256bb75b5 python-3.8.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5 - echo "16ca86fa3467e75bade26b8a9703c27f python-3.9.0-macosx10.9.pkg" >> /tmp/python_installer_checksum.md5 - md5sum -c /tmp/python_installer_checksum.md5 - time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target / - time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target / time sudo installer -pkg ./python-3.9.0-macosx10.9.pkg -target / fi @@ -129,8 +94,15 @@ then export DOTNET_CLI_TELEMETRY_OPTOUT=true fi -# PHP tests currently require using an older version of PHPUnit -ln -sf /usr/local/bin/phpunit-5.7 /usr/local/bin/phpunit +if [ "${PREPARE_BUILD_INSTALL_DEPS_PHP}" == "true" ] +then + # Install PHP 7.2 explictly to address missing php header files and + # to work with well with the pre-installed phpunit 8.4 + brew install php@7.2 + export LDFLAGS="-L/usr/local/opt/php@7.2/lib $(LDFLAGS)" + export CPPFLAGS="-I/usr/local/opt/php@7.2/include $(CPPFLAGS)" + export PATH="/usr/local/opt/php@7.2/bin:/usr/local/opt/php@7.2/sbin:$PATH" +fi # TODO(jtattermusch): better debugging of clock skew, remove once not needed date diff --git a/tools/internal_ci/macos/grpc_basictests_php.cfg b/tools/internal_ci/macos/grpc_basictests_php.cfg index 35caa84dfe0..b96bd03f976 100644 --- a/tools/internal_ci/macos/grpc_basictests_php.cfg +++ b/tools/internal_ci/macos/grpc_basictests_php.cfg @@ -15,7 +15,7 @@ # Config file for the internal CI (in protobuf text format) # Location of the continuous shell script in repository. -build_file: "grpc/tools/internal_ci/macos/grpc_run_tests_matrix.sh" +build_file: "grpc/tools/internal_ci/macos/grpc_basictests_php.sh" gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" timeout_mins: 60 action { diff --git a/tools/internal_ci/macos/grpc_basictests_php.sh b/tools/internal_ci/macos/grpc_basictests_php.sh new file mode 100644 index 00000000000..38fdbf32e73 --- /dev/null +++ b/tools/internal_ci/macos/grpc_basictests_php.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2020 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +export PREPARE_BUILD_INSTALL_DEPS_PHP=true +$(dirname $0)/grpc_run_tests_matrix.sh diff --git a/tools/internal_ci/macos/grpc_build_artifacts.sh b/tools/internal_ci/macos/grpc_build_artifacts.sh index 76ff43d3101..02b2331f5f1 100755 --- a/tools/internal_ci/macos/grpc_build_artifacts.sh +++ b/tools/internal_ci/macos/grpc_build_artifacts.sh @@ -21,6 +21,7 @@ cd $(dirname $0)/../../.. export PREPARE_BUILD_INSTALL_DEPS_CSHARP=true export PREPARE_BUILD_INSTALL_DEPS_PYTHON=true export PREPARE_BUILD_INSTALL_DEPS_RUBY=true +export PREPARE_BUILD_INSTALL_DEPS_PHP=true source tools/internal_ci/helper_scripts/prepare_build_macos_rc # install cython for all python versions diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index c2cea29f069..215abce830e 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -95,7 +95,7 @@ def create_jobspec(name, return jobspec -_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.7' +_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10' _ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} From 701828875832709a7c6ee6e830867a019b19d1eb Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Mon, 30 Nov 2020 23:33:08 -0800 Subject: [PATCH 2/4] Check sha256 --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 7f2ce40984e..58d24dff91a 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -83,6 +83,8 @@ then # Install Python 3.9 time curl -O https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg + echo "dadee1d10c1a8ed58bb64ec7409a48c9fe38599791eaaea29303ee59fb95dfeb python-3.9.0-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 + shasum -c /tmp/python_installer_checksum.sha256 time sudo installer -pkg ./python-3.9.0-macosx10.9.pkg -target / fi @@ -97,7 +99,7 @@ fi if [ "${PREPARE_BUILD_INSTALL_DEPS_PHP}" == "true" ] then # Install PHP 7.2 explictly to address missing php header files and - # to work with well with the pre-installed phpunit 8.4 + # to work well with the pre-installed phpunit 8.4 brew install php@7.2 export LDFLAGS="-L/usr/local/opt/php@7.2/lib $(LDFLAGS)" export CPPFLAGS="-I/usr/local/opt/php@7.2/include $(CPPFLAGS)" From ead79e794d9df88eb4d12bdf0040d23ad8105dc6 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Tue, 1 Dec 2020 10:17:12 -0800 Subject: [PATCH 3/4] Update prepare_build_macos_rc --- tools/internal_ci/helper_scripts/prepare_build_macos_rc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 58d24dff91a..5e5a83f0bb3 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -26,8 +26,14 @@ ulimit -a # - only add stuff that you absolutely need for your builds to work (add comment to explain why its needed) # Disable HOMEBREW update to avoid new updates which potentially have problems. +# Brew packages installed when Kokoro image was built tend to have less conflict. export HOMEBREW_NO_AUTO_UPDATE=1 +# Dump the brew configuration for debugging just in case. Check "Core tap HEAD" field +# because it should be the same as below unless it's been updated. +# - Kokoro MacOS Mojave: 27fa87c94a6cf7be40fc8f8fc96bc7c387b7781e +brew config + # Add GCP credentials for BQ access pip install --user google-api-python-client oauth2client six==1.15.0 export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json From 3a01cfd126eea3db841c77da19225dc83369f0c1 Mon Sep 17 00:00:00 2001 From: Esun Kim Date: Thu, 3 Dec 2020 15:19:20 -0800 Subject: [PATCH 4/4] Conditional python installation --- .../helper_scripts/prepare_build_macos_rc | 28 +++++++++++++++---- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 5e5a83f0bb3..477896b7d78 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -87,11 +87,29 @@ then time pip install --user virtualenv time pip install --user --upgrade Mako tox setuptools==44.1.1 twisted pyyaml pyjwt cryptography requests - # Install Python 3.9 - time curl -O https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg - echo "dadee1d10c1a8ed58bb64ec7409a48c9fe38599791eaaea29303ee59fb95dfeb python-3.9.0-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 - shasum -c /tmp/python_installer_checksum.sha256 - time sudo installer -pkg ./python-3.9.0-macosx10.9.pkg -target / + # Install Python 3.7 if it doesn't exist + if [ ! -f "/usr/local/bin/python3.7" ]; then + time curl -O https://www.python.org/ftp/python/3.7.0/python-3.7.0-macosx10.9.pkg + echo "ee4ad46ab8cd226ffc8df56d48acfdf7daa2714a9c51e6dc6262fc0b25519578 python-3.7.0-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 + shasum -c /tmp/python_installer_checksum.sha256 + time sudo installer -pkg ./python-3.7.0-macosx10.9.pkg -target / + fi + + # Install Python 3.8 if it doesn't exist + if [ ! -f "/usr/local/bin/python3.8" ]; then + time curl -O https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg + echo "30961fe060da9dc5afdc4e789a57fe9bcc0d20244474e9f095d7bfc89d2e1869 python-3.8.0-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 + shasum -c /tmp/python_installer_checksum.sha256 + time sudo installer -pkg ./python-3.8.0-macosx10.9.pkg -target / + fi + + # Install Python 3.9 if it doesn't exist + if [ ! -f "/usr/local/bin/python3.9" ]; then + time curl -O https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg + echo "dadee1d10c1a8ed58bb64ec7409a48c9fe38599791eaaea29303ee59fb95dfeb python-3.9.0-macosx10.9.pkg" > /tmp/python_installer_checksum.sha256 + shasum -c /tmp/python_installer_checksum.sha256 + time sudo installer -pkg ./python-3.9.0-macosx10.9.pkg -target / + fi fi if [ "${PREPARE_BUILD_INSTALL_DEPS_CSHARP}" == "true" ]