From 45e9707871e2e40408fa530dc6d80cf2b1a0d287 Mon Sep 17 00:00:00 2001
From: Joshua Haberman <jhaberman@gmail.com>
Date: Wed, 12 May 2021 10:42:16 -0700
Subject: [PATCH] Some more updates to PHP testing infrastructure (#8576)

* WIP.

* Added build config for all of the tests.

* Use ../src/protoc if it is available, for cases where Bazel isn't available.

* Added test_php.sh.

* Fix for the broken macOS tests.

* Move all jobs to use php80 instead of lots of separate jobs.

* Only pass -t flag if we are running in a terminal.

* Updated php_all job to use new Docker stuff.
---
 kokoro/linux/php80/build.sh         | 27 +++++++++++++--------------
 kokoro/linux/php80/continuous.cfg   |  8 +-------
 kokoro/linux/php80/presubmit.cfg    |  8 +-------
 kokoro/linux/php_all/build.sh       | 27 +++++++++++++--------------
 kokoro/linux/test_php.sh            |  6 ++++++
 kokoro/macos/prepare_build_macos_rc |  5 +++++
 php/generate_test_protos.sh         | 17 ++++++++++++++---
 php/prepare_c_extension.sh          | 18 ++++++++++++++++--
 php/tests/compile_extension.sh      |  5 ++++-
 9 files changed, 73 insertions(+), 48 deletions(-)
 create mode 100755 kokoro/linux/test_php.sh

diff --git a/kokoro/linux/php80/build.sh b/kokoro/linux/php80/build.sh
index f17aec104f..ba269e52db 100755
--- a/kokoro/linux/php80/build.sh
+++ b/kokoro/linux/php80/build.sh
@@ -1,18 +1,17 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
-# running the "pull request" project:
-#
-# This script selects a specific Dockerfile (for building a Docker image) and
-# a script to run inside that image.  Then we delegate to the general
-# build_and_run_docker.sh script.
+# This is the entry point for kicking off a Kokoro job. This path is referenced
+# from the .cfg files in this directory.
+
+set -ex
+
+cd $(dirname $0)
 
-# Change to repo root
-cd $(dirname $0)/../../..
+# Most of our tests use a debug build of PHP, but we do one build against an opt
+# php just in case that surfaces anything unexpected.
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-14a06550010c0649bf69b6c9b803c1ca609bbb6d
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/php80
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
-export OUTPUT_DIR=testoutput
-export TEST_SET="php8.0_all"
-./kokoro/linux/build_and_run_docker.sh
+../test_php.sh gcr.io/protobuf-build/php/linux:7.0.33-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
+../test_php.sh gcr.io/protobuf-build/php/linux:7.3.28-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
+../test_php.sh gcr.io/protobuf-build/php/linux:7.4.18-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
diff --git a/kokoro/linux/php80/continuous.cfg b/kokoro/linux/php80/continuous.cfg
index 8426318bbe..6d67a8c31a 100644
--- a/kokoro/linux/php80/continuous.cfg
+++ b/kokoro/linux/php80/continuous.cfg
@@ -2,10 +2,4 @@
 
 # Location of the build script in repository
 build_file: "protobuf/kokoro/linux/php80/build.sh"
-timeout_mins: 120
-
-action {
-  define_artifacts {
-    regex: "**/sponge_log.xml"
-  }
-}
+timeout_mins: 20
diff --git a/kokoro/linux/php80/presubmit.cfg b/kokoro/linux/php80/presubmit.cfg
index 8426318bbe..6d67a8c31a 100644
--- a/kokoro/linux/php80/presubmit.cfg
+++ b/kokoro/linux/php80/presubmit.cfg
@@ -2,10 +2,4 @@
 
 # Location of the build script in repository
 build_file: "protobuf/kokoro/linux/php80/build.sh"
-timeout_mins: 120
-
-action {
-  define_artifacts {
-    regex: "**/sponge_log.xml"
-  }
-}
+timeout_mins: 20
diff --git a/kokoro/linux/php_all/build.sh b/kokoro/linux/php_all/build.sh
index 23468a7010..ba269e52db 100755
--- a/kokoro/linux/php_all/build.sh
+++ b/kokoro/linux/php_all/build.sh
@@ -1,18 +1,17 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
-# running the "pull request" project:
-#
-# This script selects a specific Dockerfile (for building a Docker image) and
-# a script to run inside that image.  Then we delegate to the general
-# build_and_run_docker.sh script.
+# This is the entry point for kicking off a Kokoro job. This path is referenced
+# from the .cfg files in this directory.
+
+set -ex
+
+cd $(dirname $0)
 
-# Change to repo root
-cd $(dirname $0)/../../..
+# Most of our tests use a debug build of PHP, but we do one build against an opt
+# php just in case that surfaces anything unexpected.
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-14a06550010c0649bf69b6c9b803c1ca609bbb6d
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/php
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
-export OUTPUT_DIR=testoutput
-export TEST_SET="php_all"
-./kokoro/linux/build_and_run_docker.sh
+../test_php.sh gcr.io/protobuf-build/php/linux:7.0.33-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
+../test_php.sh gcr.io/protobuf-build/php/linux:7.3.28-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
+../test_php.sh gcr.io/protobuf-build/php/linux:7.4.18-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
diff --git a/kokoro/linux/test_php.sh b/kokoro/linux/test_php.sh
new file mode 100755
index 0000000000..918dd6d258
--- /dev/null
+++ b/kokoro/linux/test_php.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+
+test -t 1 && USE_TTY="-it" 
+docker run ${USE_TTY} -v$(realpath $(dirname $0)/../..):/workspace $1 "composer test && composer test_c"
diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc
index d94dd543fd..dcd17f330c 100755
--- a/kokoro/macos/prepare_build_macos_rc
+++ b/kokoro/macos/prepare_build_macos_rc
@@ -45,6 +45,11 @@ sudo rm -rf \
 git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow
 git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask fetch --unshallow
 
+# This is needed to fix a conflict between the ilmbase and imath packages,
+# which seem to conflict with each other by both trying to install
+# libImath.dylib.
+brew unlink ilmbase
+
 brew update
 brew upgrade
 
diff --git a/php/generate_test_protos.sh b/php/generate_test_protos.sh
index 0aa4bdbe43..e86ca07a68 100755
--- a/php/generate_test_protos.sh
+++ b/php/generate_test_protos.sh
@@ -4,7 +4,17 @@ set -e
 
 cd `dirname $0`
 
-if [[ -d tmp && -z $(find tests/proto ../src/protoc -newer tmp) ]]; then
+./prepare_c_extension.sh
+
+if ../src/protoc --help > /dev/null; then
+  PROTOC=src/protoc
+else
+  (cd .. && bazel build -c opt :protoc)
+  PROTOC=bazel-bin/protoc
+fi
+
+
+if [[ -d tmp && -z $(find tests/proto ../$PROTOC -newer tmp) ]]; then
   # Generated protos are already present and up to date, so we can skip protoc.
   #
   # Protoc is very fast, but sometimes it is not available (like if we haven't
@@ -16,12 +26,13 @@ fi
 rm -rf tmp
 mkdir -p tmp
 
-find tests/proto -type f -name "*.proto"| xargs ../src/protoc --php_out=tmp -I../src -Itests
+cd ..
+find php/tests/proto -type f -name "*.proto"| xargs $PROTOC --php_out=php/tmp -Isrc -Iphp/tests
 
 if [ "$1" = "--aggregate_metadata" ]; then
   # Overwrite some of the files to use aggregation.
   AGGREGATED_FILES="tests/proto/test.proto tests/proto/test_include.proto tests/proto/test_import_descriptor_proto.proto"
-  ../src/protoc --php_out=aggregate_metadata=foo#bar:tmp -I../src -Itests $AGGREGATED_FILES
+  $PROTOC --php_out=aggregate_metadata=foo#bar:php/tmp -Isrc -Iphp/tests $AGGREGATED_FILES
 fi
 
 echo "Generated test protos from tests/proto -> tmp"
diff --git a/php/prepare_c_extension.sh b/php/prepare_c_extension.sh
index 84cd1aa3d1..1b6b9f1dae 100755
--- a/php/prepare_c_extension.sh
+++ b/php/prepare_c_extension.sh
@@ -1,6 +1,20 @@
 
+cd $(dirname $0)
+
+if [[ -f ext/google/protobuf/third_party/wyhash/wyhash.h && -z $(find ../third_party/wyhash -newer ext/google/protobuf/third_party) ]]; then
+  # Generated protos are already present and up to date, so we can skip protoc.
+  #
+  # Protoc is very fast, but sometimes it is not available (like if we haven't
+  # built it in Docker). Skipping it helps us proceed in this case.
+  echo "wyhash is up to date, skipping."
+  exit 0
+fi
+
 # wyhash has to live in the base third_party directory.
 # We copy it into the ext/google/protobuf directory for the build
 # (and for the release to PECL).
-mkdir -p ../ext/google/protobuf/third_party/wyhash
-cp ../../third_party/wyhash/* ../ext/google/protobuf/third_party/wyhash
+rm -rf ext/google/protobuf/third_party
+mkdir -p ext/google/protobuf/third_party/wyhash
+cp ../third_party/wyhash/* ext/google/protobuf/third_party/wyhash
+
+echo "Copied wyhash from ../third_party -> ext/google/protobuf/third_party"
diff --git a/php/tests/compile_extension.sh b/php/tests/compile_extension.sh
index d26fcb472e..fe77bbc3a4 100755
--- a/php/tests/compile_extension.sh
+++ b/php/tests/compile_extension.sh
@@ -13,13 +13,16 @@ if [ "$1" != "--release" ]; then
   CONFIGURE_OPTIONS+=("CFLAGS=-g -O0 -Wall")
 fi
 
+FINGERPRINT="$(sha256sum $(which php)) ${CONFIGURE_OPTIONS[@]}"
+
 # If the PHP interpreter we are building against or the arguments
 # have changed, we must regenerated the Makefile.
-if [[ ! -f Makefile ]] || [[ "$(grep '  \$ ./configure' config.log)" != "  $ ${CONFIGURE_OPTIONS[@]}" ]]; then
+if [[ ! -f BUILD_STAMP ]] || [[ "$(cat BUILD_STAMP)" != "$FINGERPRINT" ]]; then
   phpize --clean
   rm -f configure.in configure.ac
   phpize
   "${CONFIGURE_OPTIONS[@]}"
+  echo "$FINGERPRINT" > BUILD_STAMP
 fi
 
 make