From 2fffff8320a7afd0979300b2a9baadc72ca82f53 Mon Sep 17 00:00:00 2001 From: AN Long Date: Thu, 23 Sep 2021 06:09:39 +0800 Subject: [PATCH 1/7] Add python-requires in setup.py (#8989) * Add python-requires in setup.py * Update setup.py --- python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/setup.py b/python/setup.py index f639732baf..50fd5be4a7 100755 --- a/python/setup.py +++ b/python/setup.py @@ -308,4 +308,5 @@ if __name__ == '__main__': }, install_requires=install_requires, ext_modules=ext_module_list, + python_requires='>=3.5', ) From 151e632bac75f6179043655ae54aa6f835508e72 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 23 Sep 2021 15:08:32 -0700 Subject: [PATCH 2/7] Fix dist install test by ensuring that we use Python 3 (#9011) * Fix dist install test by ensuring that we use Python 3 Now that we have dropped Python 2 support, we need to make sure this install test uses Python 3. * Update Docker image to install Python 3 version of setuptools * Run pip3 instead of pip --- kokoro/linux/dockerfile/test/java_stretch/Dockerfile | 4 ++-- tests.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kokoro/linux/dockerfile/test/java_stretch/Dockerfile b/kokoro/linux/dockerfile/test/java_stretch/Dockerfile index 3e72046f47..b9f562a2b4 100644 --- a/kokoro/linux/dockerfile/test/java_stretch/Dockerfile +++ b/kokoro/linux/dockerfile/test/java_stretch/Dockerfile @@ -24,7 +24,7 @@ RUN apt-get update && apt-get install -y \ maven \ openjdk-8-jdk \ # Python dependencies - python-setuptools \ - python-pip \ + python3-setuptools \ + python3-pip \ virtualenv \ && apt-get clean diff --git a/tests.sh b/tests.sh index 1955b7bdf9..5dc2eb6ff8 100755 --- a/tests.sh +++ b/tests.sh @@ -112,8 +112,8 @@ build_dist_install() { virtualenv --no-site-packages venv source venv/bin/activate pushd python - python setup.py clean build sdist - pip install dist/protobuf-*.tar.gz + python3 setup.py clean build sdist + pip3 install dist/protobuf-*.tar.gz popd deactivate rm -rf python/venv From 0e7a35f24bb3eef02bd657aae867ed58ca5a74e3 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 29 Sep 2021 13:41:33 -0700 Subject: [PATCH 3/7] Update dist_install test to work around Python issue googletest uses a Python script in its build which is not compatible with Python 3. Unfortunately we can't easily upgrade googletest right now, so this commit works around the problem by putting a python symlink pointing to Python 2 in the $PATH. --- tests.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests.sh b/tests.sh index 5dc2eb6ff8..71635f9f7f 100755 --- a/tests.sh +++ b/tests.sh @@ -88,6 +88,18 @@ build_cpp_distcheck() { } build_dist_install() { + # Create a symlink pointing to python2 and put it at the beginning of $PATH. + # This is necessary because the googletest build system involves a Python + # script that is not compatible with Python 3. More recent googletest + # versions have fixed this, but they have also removed the autotools build + # system support that we rely on. This is a temporary workaround to keep the + # googletest build working when the default python binary is Python 3. + mkdir tmp || true + pushd tmp + ln -s /usr/bin/python2 ./python + popd + PATH=$PWD/tmp:$PATH + # Initialize any submodules. git submodule update --init --recursive ./autogen.sh From 0333cb77de63e255eb6f07bb496ad03c90ed7d5e Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 24 Sep 2021 14:43:58 -0700 Subject: [PATCH 4/7] Update ruby/travis-test.sh to print commands --- ruby/travis-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index 8980395f78..965782a615 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Exit on any error. -set -e +set -ex test_version() { version=$1 From 55b0e11f528e41dc96ad5902fb5abc5b153486c9 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 30 Sep 2021 13:16:53 -0700 Subject: [PATCH 5/7] Fix Ruby tests on Mac It appears that this extra conditional in travis-test.sh is no longer necessary, and in fact we need to avoid using a custom version of OpenSSL since that is causing its own error. --- ruby/travis-test.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/ruby/travis-test.sh b/ruby/travis-test.sh index 965782a615..b57d8b2db9 100755 --- a/ruby/travis-test.sh +++ b/ruby/travis-test.sh @@ -18,7 +18,7 @@ test_version() { rake gc_test && cd ../conformance && make test_jruby && cd ../ruby/compatibility_tests/v3.0.0 && ./test.sh" - elif [ "$version" == "ruby-2.6.0" -o "$version" == "ruby-2.7.0" -o "$version" == "ruby-3.0.2" ] ; then + else bash --login -c \ "rvm install $version && rvm use $version && \ which ruby && \ @@ -29,19 +29,6 @@ test_version() { cd ../conformance && make ${RUBY_CONFORMANCE} && cd ../ruby/compatibility_tests/v3.0.0 && cp -R ../../lib lib && ./test.sh" - else - # Recent versions of OSX have deprecated OpenSSL, so we have to explicitly - # provide a path to the OpenSSL directory installed via Homebrew. - bash --login -c \ - "rvm install $version --with-openssl-dir=`brew --prefix openssl` && \ - rvm use $version && \ - which ruby && \ - git clean -f && \ - gem install bundler -v 1.17.3 && bundle && \ - rake test && - rake gc_test && - cd ../conformance && make ${RUBY_CONFORMANCE} && - cd ../ruby/compatibility_tests/v3.0.0 && ./test.sh" fi } From 5378c9a7105d8d2a69b9e587797e1223bb0986d0 Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Thu, 30 Sep 2021 14:39:12 -0700 Subject: [PATCH 6/7] Use the stable version of rvm I am hoping this will fix the error we are getting as described here: https://github.com/rvm/rvm/issues/5014 --- kokoro/macos/prepare_build_macos_rc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index dcd17f330c..2db706b12c 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -85,5 +85,5 @@ if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then curl -sSL https://rvm.io/mpapis.asc | gpg --import - curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - - curl -sSL https://get.rvm.io | bash -s master --ruby + curl -sSL https://get.rvm.io | bash -s stable --ruby fi From aeb624d3b4028fb6ec2f6d76b0d9a43dc2e7e87b Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Fri, 1 Oct 2021 15:45:55 -0700 Subject: [PATCH 7/7] Avoid installing anything from Homebrew in test runs (#9043) Installing and upgrading Homebrew packages is taking quite a lot of time (around 20-30 minutes) for each run. This commit removes all Homebrew usage from the test runs. Homebrew may have been necessary at some point in the past, but now it appears that everything works without it. The preinstalled build tools seem to be sufficient for building protoc, and Ruby is something we already get from rvm. --- .../objectivec_cocoapods_integration/build.sh | 1 - kokoro/macos/prepare_build_macos_rc | 52 ------------------- kokoro/macos/ruby23/build.sh | 1 - kokoro/macos/ruby24/build.sh | 1 - kokoro/macos/ruby25/build.sh | 1 - kokoro/macos/ruby26/build.sh | 1 - kokoro/macos/ruby27/build.sh | 1 - kokoro/macos/ruby30/build.sh | 1 - 8 files changed, 59 deletions(-) diff --git a/kokoro/macos/objectivec_cocoapods_integration/build.sh b/kokoro/macos/objectivec_cocoapods_integration/build.sh index 8f3c9b4c15..f96d2899d9 100755 --- a/kokoro/macos/objectivec_cocoapods_integration/build.sh +++ b/kokoro/macos/objectivec_cocoapods_integration/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_COCOAPODS=yes source kokoro/macos/prepare_build_macos_rc ./tests.sh objectivec_cocoapods_integration diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index 2db706b12c..ef428fcda3 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -19,58 +19,6 @@ export DEVELOPER_DIR=/Applications/Xcode_11.3.app/Contents/Developer export CC=gcc export CXX=g++ -## -# Brew: update, then upgrade the installed tools to current version and install -# some needed ones not in the Kokoro base image. This ensure current versions -# of CMake, autotools, etc. - -# But first... -# -# The transitive deps of the installed tools need protobuf, but Kokoro manually -# installed it outside of brew so it needs to be removed so brew can install the -# tools (and a newer version of protobuf). g/kokoro-users/7FRvQMUdN40 about why -# it is a manual install vs. a brew install in the first place. -sudo rm -rf \ - /usr/local/include/google/protobuf \ - /usr/local/bin/protoc -# Likewise, updating python can have issues because of some existing binaries. -sudo rm -rf \ - /usr/local/bin/2to3* \ - /usr/local/bin/idle3* \ - /usr/local/bin/pip3 \ - /usr/local/bin/pydoc3* \ - /usr/local/bin/python3* \ - /usr/local/bin/pyvenv* - -git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow -git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask fetch --unshallow - -# This is needed to fix a conflict between the ilmbase and imath packages, -# which seem to conflict with each other by both trying to install -# libImath.dylib. -brew unlink ilmbase - -brew update -brew upgrade - -## -# Install Ruby - -if [[ "${KOKORO_INSTALL_RUBY:-}" == "yes" ]] ; then - brew install ruby -fi - -## -# Install Cocoapods - -if [[ "${KOKORO_INSTALL_COCOAPODS:-}" == "yes" ]] ; then - # The existing cocoapods was installed via gem, but that doesn't work well - # with the overlap in deps with things managed by brew (errors around ruby - # versions, etc.); so remove it and install in via brew instead. - gem uninstall -a "$(gem list | grep cocoapods | cut -d ' ' -f 1)" - brew install cocoapods -fi - ## # Install Tox diff --git a/kokoro/macos/ruby23/build.sh b/kokoro/macos/ruby23/build.sh index 9317838781..5a29a997fc 100755 --- a/kokoro/macos/ruby23/build.sh +++ b/kokoro/macos/ruby23/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_RUBY=yes KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc diff --git a/kokoro/macos/ruby24/build.sh b/kokoro/macos/ruby24/build.sh index 51bb2e603b..10ac85f7a4 100755 --- a/kokoro/macos/ruby24/build.sh +++ b/kokoro/macos/ruby24/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_RUBY=yes KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc diff --git a/kokoro/macos/ruby25/build.sh b/kokoro/macos/ruby25/build.sh index ba2d0a4d19..48c894081b 100755 --- a/kokoro/macos/ruby25/build.sh +++ b/kokoro/macos/ruby25/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_RUBY=yes KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc diff --git a/kokoro/macos/ruby26/build.sh b/kokoro/macos/ruby26/build.sh index 5a4c243222..1b94fe1b55 100755 --- a/kokoro/macos/ruby26/build.sh +++ b/kokoro/macos/ruby26/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_RUBY=yes KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc diff --git a/kokoro/macos/ruby27/build.sh b/kokoro/macos/ruby27/build.sh index b1529b9da0..baebdb792a 100755 --- a/kokoro/macos/ruby27/build.sh +++ b/kokoro/macos/ruby27/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_RUBY=yes KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc diff --git a/kokoro/macos/ruby30/build.sh b/kokoro/macos/ruby30/build.sh index 6b9bfb3215..b1e0641c96 100755 --- a/kokoro/macos/ruby30/build.sh +++ b/kokoro/macos/ruby30/build.sh @@ -6,7 +6,6 @@ cd $(dirname $0)/../../.. # Prepare worker environment to run tests -KOKORO_INSTALL_RUBY=yes KOKORO_INSTALL_RVM=yes source kokoro/macos/prepare_build_macos_rc