Some more updates to PHP testing infrastructure (#8576)

* WIP.

* Added build config for all of the tests.

* Use ../src/protoc if it is available, for cases where Bazel isn't available.

* Added test_php.sh.

* Fix for the broken macOS tests.

* Move all jobs to use php80 instead of lots of separate jobs.

* Only pass -t flag if we are running in a terminal.

* Updated php_all job to use new Docker stuff.

kokoro/linux/php80/build.sh

@@ -1,18 +1,17 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
+# This is the entry point for kicking off a Kokoro job. This path is referenced
-# running the "pull request" project:
+# from the .cfg files in this directory.
-#
+
-# This script selects a specific Dockerfile (for building a Docker image) and
+set -ex
-# a script to run inside that image.  Then we delegate to the general
+
-# build_and_run_docker.sh script.
+cd $(dirname $0)
 
-# Change to repo root
+# Most of our tests use a debug build of PHP, but we do one build against an opt
-cd $(dirname $0)/../../..
+# php just in case that surfaces anything unexpected.
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-14a06550010c0649bf69b6c9b803c1ca609bbb6d
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
+../test_php.sh gcr.io/protobuf-build/php/linux:7.0.33-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/php80
+../test_php.sh gcr.io/protobuf-build/php/linux:7.3.28-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+../test_php.sh gcr.io/protobuf-build/php/linux:7.4.18-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export OUTPUT_DIR=testoutput
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export TEST_SET="php8.0_all"
-./kokoro/linux/build_and_run_docker.sh

kokoro/linux/php80/continuous.cfg

@@ -2,10 +2,4 @@
 
 # Location of the build script in repository
 build_file: "protobuf/kokoro/linux/php80/build.sh"
-timeout_mins: 120
+timeout_mins: 20
-
-action {
-  define_artifacts {
-    regex: "**/sponge_log.xml"
-  }
-}

kokoro/linux/php80/presubmit.cfg

@@ -2,10 +2,4 @@
 
 # Location of the build script in repository
 build_file: "protobuf/kokoro/linux/php80/build.sh"
-timeout_mins: 120
+timeout_mins: 20
-
-action {
-  define_artifacts {
-    regex: "**/sponge_log.xml"
-  }
-}

kokoro/linux/php_all/build.sh

@@ -1,18 +1,17 @@
 #!/bin/bash
 #
-# This is the top-level script we give to Kokoro as the entry point for
+# This is the entry point for kicking off a Kokoro job. This path is referenced
-# running the "pull request" project:
+# from the .cfg files in this directory.
-#
+
-# This script selects a specific Dockerfile (for building a Docker image) and
+set -ex
-# a script to run inside that image.  Then we delegate to the general
+
-# build_and_run_docker.sh script.
+cd $(dirname $0)
 
-# Change to repo root
+# Most of our tests use a debug build of PHP, but we do one build against an opt
-cd $(dirname $0)/../../..
+# php just in case that surfaces anything unexpected.
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-14a06550010c0649bf69b6c9b803c1ca609bbb6d
 
-export DOCKERHUB_ORGANIZATION=protobuftesting
+../test_php.sh gcr.io/protobuf-build/php/linux:7.0.33-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/php
+../test_php.sh gcr.io/protobuf-build/php/linux:7.3.28-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh
+../test_php.sh gcr.io/protobuf-build/php/linux:7.4.18-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export OUTPUT_DIR=testoutput
+../test_php.sh gcr.io/protobuf-build/php/linux:8.0.5-dbg-14a06550010c0649bf69b6c9b803c1ca609bbb6d
-export TEST_SET="php_all"
-./kokoro/linux/build_and_run_docker.sh

kokoro/linux/test_php.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -ex
+
+test -t 1 && USE_TTY="-it" 
+docker run ${USE_TTY} -v$(realpath $(dirname $0)/../..):/workspace $1 "composer test && composer test_c"

kokoro/macos/prepare_build_macos_rc

@@ -45,6 +45,11 @@ sudo rm -rf \
 git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-core fetch --unshallow
 git -C /usr/local/Homebrew/Library/Taps/homebrew/homebrew-cask fetch --unshallow
 
+# This is needed to fix a conflict between the ilmbase and imath packages,
+# which seem to conflict with each other by both trying to install
+# libImath.dylib.
+brew unlink ilmbase
+
 brew update
 brew upgrade
 

php/generate_test_protos.sh

@@ -4,7 +4,17 @@ set -e
 
 cd `dirname $0`
 
-if [[ -d tmp && -z $(find tests/proto ../src/protoc -newer tmp) ]]; then
+./prepare_c_extension.sh
+
+if ../src/protoc --help > /dev/null; then
+  PROTOC=src/protoc
+else
+  (cd .. && bazel build -c opt :protoc)
+  PROTOC=bazel-bin/protoc
+fi
+
+
+if [[ -d tmp && -z $(find tests/proto ../$PROTOC -newer tmp) ]]; then
   # Generated protos are already present and up to date, so we can skip protoc.
   #
   # Protoc is very fast, but sometimes it is not available (like if we haven't
@@ -16,12 +26,13 @@ fi
 rm -rf tmp
 mkdir -p tmp
 
-find tests/proto -type f -name "*.proto"| xargs ../src/protoc --php_out=tmp -I../src -Itests
+cd ..
+find php/tests/proto -type f -name "*.proto"| xargs $PROTOC --php_out=php/tmp -Isrc -Iphp/tests
 
 if [ "$1" = "--aggregate_metadata" ]; then
   # Overwrite some of the files to use aggregation.
   AGGREGATED_FILES="tests/proto/test.proto tests/proto/test_include.proto tests/proto/test_import_descriptor_proto.proto"
-  ../src/protoc --php_out=aggregate_metadata=foo#bar:tmp -I../src -Itests $AGGREGATED_FILES
+  $PROTOC --php_out=aggregate_metadata=foo#bar:php/tmp -Isrc -Iphp/tests $AGGREGATED_FILES
 fi
 
 echo "Generated test protos from tests/proto -> tmp"

php/prepare_c_extension.sh

@@ -1,6 +1,20 @@
 
+cd $(dirname $0)
+
+if [[ -f ext/google/protobuf/third_party/wyhash/wyhash.h && -z $(find ../third_party/wyhash -newer ext/google/protobuf/third_party) ]]; then
+  # Generated protos are already present and up to date, so we can skip protoc.
+  #
+  # Protoc is very fast, but sometimes it is not available (like if we haven't
+  # built it in Docker). Skipping it helps us proceed in this case.
+  echo "wyhash is up to date, skipping."
+  exit 0
+fi
+
 # wyhash has to live in the base third_party directory.
 # We copy it into the ext/google/protobuf directory for the build
 # (and for the release to PECL).
-mkdir -p ../ext/google/protobuf/third_party/wyhash
+rm -rf ext/google/protobuf/third_party
-cp ../../third_party/wyhash/* ../ext/google/protobuf/third_party/wyhash
+mkdir -p ext/google/protobuf/third_party/wyhash
+cp ../third_party/wyhash/* ext/google/protobuf/third_party/wyhash
+
+echo "Copied wyhash from ../third_party -> ext/google/protobuf/third_party"

php/tests/compile_extension.sh

@@ -13,13 +13,16 @@ if [ "$1" != "--release" ]; then
   CONFIGURE_OPTIONS+=("CFLAGS=-g -O0 -Wall")
 fi
 
+FINGERPRINT="$(sha256sum $(which php)) ${CONFIGURE_OPTIONS[@]}"
+
 # If the PHP interpreter we are building against or the arguments
 # have changed, we must regenerated the Makefile.
-if [[ ! -f Makefile ]] || [[ "$(grep '  \$ ./configure' config.log)" != "  $ ${CONFIGURE_OPTIONS[@]}" ]]; then
+if [[ ! -f BUILD_STAMP ]] || [[ "$(cat BUILD_STAMP)" != "$FINGERPRINT" ]]; then
   phpize --clean
   rm -f configure.in configure.ac
   phpize
   "${CONFIGURE_OPTIONS[@]}"
+  echo "$FINGERPRINT" > BUILD_STAMP
 fi
 
 make