From ce8b7af46b82567c077611bae2e1285e04cfa7f2 Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Fri, 5 Jun 2020 16:35:54 -0400 Subject: [PATCH] Update the brew workflow - Remove the `brew install`; the kokoro image already has it, just update/upgrade instead. - Remove `prune`, logs had: """ Error: Unknown command: prune """ - Remove `uninstall`, logs had: """ Error: Refusing to uninstall /usr/local/Cellar/cmake/3.16.4 and /usr/local/Cellar/icu4c/64.2 because they are required by ceres-solver, ffmpeg, harfbuzz, libass and opencv, which are currently installed. You can override this and force removal with: brew uninstall --ignore-dependencies node icu4c cmake wget """ - Skip installing some things since they are already in the base image, logs had: """ Warning: gflags 2.2.2 is already installed and up-to-date To reinstall 2.2.2, run `brew reinstall gflags` Warning: openssl@1.1 1.1.1g is already installed and up-to-date To reinstall 1.1.1g, run `brew reinstall openssl@1.1` Warning: pcre 8.44 is already installed and up-to-date To reinstall 8.44, run `brew reinstall pcre` """ - Don't install gpg gpg2 as gnupg is already installed, also use gpg instead of gpg2 for commands (and update the commands), logs had: """ kokoro/macos/prepare_build_macos_rc: line 44: gpg2: command not found kokoro/macos/prepare_build_macos_rc: line 45: gpg2: command not found """ - Add env guards to control all the option installs and only request them be installed in the cases that need it. This avoids having to install/update the things like ruby when some other tool only needed in some configs is install differently and could have conflicts. - Switch to brew for cocoapods to avoid compat issues on the supporting libraries. --- .../objectivec_cocoapods_integration/build.sh | 1 + kokoro/macos/prepare_build_macos_rc | 63 +++++++++++++++---- kokoro/macos/python/build.sh | 1 + kokoro/macos/python_cpp/build.sh | 1 + kokoro/macos/ruby23/build.sh | 2 + kokoro/macos/ruby24/build.sh | 2 + kokoro/macos/ruby25/build.sh | 2 + kokoro/macos/ruby26/build.sh | 2 + kokoro/macos/ruby27/build.sh | 2 + tests.sh | 2 - 10 files changed, 64 insertions(+), 14 deletions(-) diff --git a/kokoro/macos/objectivec_cocoapods_integration/build.sh b/kokoro/macos/objectivec_cocoapods_integration/build.sh index f96d2899d9..8f3c9b4c15 100755 --- a/kokoro/macos/objectivec_cocoapods_integration/build.sh +++ b/kokoro/macos/objectivec_cocoapods_integration/build.sh @@ -6,6 +6,7 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_COCOAPODS=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh objectivec_cocoapods_integration diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index 2428750a8a..18ce60b17f 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -2,6 +2,8 @@ # # This script sets up a Kokoro MacOS worker for running Protobuf tests +set -eux + ## # Select Xcode version @@ -18,24 +20,61 @@ export CC=gcc export CXX=g++ ## -# Install Brew and core softwares +# Brew: update, then upgrade the installed tools to current version and install +# some needed ones not in the Kokoro base image. This ensure current versions +# of CMake, autotools, etc. + +# But first... +# +# The transitive deps of the installed tools need protobuf, but Kokoro manually +# installed it outside of brew so it needs to be removed so brew can install the +# tools (and a newer version of protobuf). g/kokoro-users/7FRvQMUdN40 about why +# it is a manual install vs. a brew install in the first place. +sudo rm -rf \ + /usr/local/include/google/protobuf \ + /usr/local/bin/protoc +# Likewise, updating python can have issues because of some existing binaries. +sudo rm -rf \ + /usr/local/bin/2to3* \ + /usr/local/bin/idle3* \ + /usr/local/bin/pydoc3* \ + /usr/local/bin/python3* \ + /usr/local/bin/pyvenv* + +brew update +brew upgrade + +## +# Install Ruby -ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)" -source $HOME/.rvm/scripts/rvm -brew uninstall node icu4c cmake wget -brew prune -brew install gflags gpg gpg2 node openssl pcre ruby cmake wget -sudo chown -R $(whoami) /usr/local -brew postinstall node +if [[ "${KOKORO_INSTALL_RUBY:-}" == "yes" ]] ; then + brew install ruby +fi + +## +# Install Cocoapods + +if [[ "${KOKORO_INSTALL_COCOAPODS:-}" == "yes" ]] ; then + # The existing cocoapods was installed via gem, but that doesn't work well + # with the overlap in deps with things managed by brew (errors around ruby + # versions, etc.); so remove it and install in via brew instead. + gem uninstall -a "$(gem list | grep cocoapods | cut -d ' ' -f 1)" + brew install cocoapods +fi ## # Install Tox -sudo pip install tox==2.4.1 +if [[ "${KOKORO_INSTALL_TOX:-}" == "yes" ]] ; then + sudo pip install tox==2.4.1 +fi ## # Install RVM -gpg2 --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - -curl -sSL https://get.rvm.io | bash -s stable --ruby +if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then + curl -sSL https://rvm.io/mpapis.asc | gpg --import - + curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - + + curl -sSL https://get.rvm.io | bash -s stable --ruby +fi diff --git a/kokoro/macos/python/build.sh b/kokoro/macos/python/build.sh index 6b17b95446..388e24b4ac 100755 --- a/kokoro/macos/python/build.sh +++ b/kokoro/macos/python/build.sh @@ -6,6 +6,7 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_TOX=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh python diff --git a/kokoro/macos/python_cpp/build.sh b/kokoro/macos/python_cpp/build.sh index cb53def9d3..f86dd6f76e 100755 --- a/kokoro/macos/python_cpp/build.sh +++ b/kokoro/macos/python_cpp/build.sh @@ -6,6 +6,7 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_TOX=yes source kokoro/macos/prepare_build_macos_rc g++ --version diff --git a/kokoro/macos/ruby23/build.sh b/kokoro/macos/ruby23/build.sh index dc950a20d4..9317838781 100755 --- a/kokoro/macos/ruby23/build.sh +++ b/kokoro/macos/ruby23/build.sh @@ -6,6 +6,8 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_RUBY=yes +KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh ruby23 diff --git a/kokoro/macos/ruby24/build.sh b/kokoro/macos/ruby24/build.sh index 3c3a190d9d..51bb2e603b 100755 --- a/kokoro/macos/ruby24/build.sh +++ b/kokoro/macos/ruby24/build.sh @@ -6,6 +6,8 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_RUBY=yes +KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh ruby24 diff --git a/kokoro/macos/ruby25/build.sh b/kokoro/macos/ruby25/build.sh index 38e90aa75c..ba2d0a4d19 100755 --- a/kokoro/macos/ruby25/build.sh +++ b/kokoro/macos/ruby25/build.sh @@ -6,6 +6,8 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_RUBY=yes +KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh ruby25 diff --git a/kokoro/macos/ruby26/build.sh b/kokoro/macos/ruby26/build.sh index 6c33ed0a7e..5a4c243222 100755 --- a/kokoro/macos/ruby26/build.sh +++ b/kokoro/macos/ruby26/build.sh @@ -6,6 +6,8 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_RUBY=yes +KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh ruby26 diff --git a/kokoro/macos/ruby27/build.sh b/kokoro/macos/ruby27/build.sh index 16bcbd6cbd..b1529b9da0 100755 --- a/kokoro/macos/ruby27/build.sh +++ b/kokoro/macos/ruby27/build.sh @@ -6,6 +6,8 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests +KOKORO_INSTALL_RUBY=yes +KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh ruby27 diff --git a/tests.sh b/tests.sh index 0a0e7988fc..eee3e99e96 100755 --- a/tests.sh +++ b/tests.sh @@ -305,8 +305,6 @@ build_objectivec_tvos_release() { } build_objectivec_cocoapods_integration() { - # Update pod to the latest version. - gem install cocoapods --no_document objectivec/Tests/CocoaPods/run_tests.sh }