From ce5b4e949fc75ed4c19e9ccfec7dc95c8ee9ae45 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 31 Aug 2021 09:23:13 +0200 Subject: [PATCH] Faster and streamlined PHP distribtests (for linux and macos) (#27101) * faster and streamlined PHP distribtests * add kokoro job for PHP distribtests * php distribtests job for macos * add trailing newline --- .../linux/grpc_distribtests_php.cfg | 26 ++++++++++ .../linux/grpc_distribtests_php.sh | 51 +++++++++++++++++++ .../macos/grpc_distribtests_php.cfg | 27 ++++++++++ .../macos/grpc_distribtests_php.sh | 47 +++++++++++++++++ tools/run_tests/artifacts/artifact_targets.py | 16 ++++-- .../run_tests/artifacts/build_artifact_php.sh | 7 --- .../run_tests/artifacts/build_package_php.sh | 2 +- .../artifacts/distribtest_targets.py | 2 +- 8 files changed, 164 insertions(+), 14 deletions(-) create mode 100644 tools/internal_ci/linux/grpc_distribtests_php.cfg create mode 100755 tools/internal_ci/linux/grpc_distribtests_php.sh create mode 100644 tools/internal_ci/macos/grpc_distribtests_php.cfg create mode 100644 tools/internal_ci/macos/grpc_distribtests_php.sh diff --git a/tools/internal_ci/linux/grpc_distribtests_php.cfg b/tools/internal_ci/linux/grpc_distribtests_php.cfg new file mode 100644 index 00000000000..bdf31484519 --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests_php.cfg @@ -0,0 +1,26 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_distribtests_php.sh" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/linux/grpc_distribtests_php.sh b/tools/internal_ci/linux/grpc_distribtests_php.sh new file mode 100755 index 00000000000..a0de6701d43 --- /dev/null +++ b/tools/internal_ci/linux/grpc_distribtests_php.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_linux_rc + +# Build all PHP linux artifacts +tools/run_tests/task_runner.py -f artifact linux php -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" + +# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/* input_artifacts/ || true +rm -rf artifacts_from_build_artifacts_step +mv artifacts artifacts_from_build_artifacts_step || true + +# This step mostly just copies artifacts from input_artifacts (but it also does some wheel stripping) +tools/run_tests/task_runner.py -f package linux php -x build_packages/sponge_log.xml || FAILED="true" + +# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +# in addition to that, preserve the contents of "artifacts" directory since we want kokoro +# to upload its contents as job output artifacts +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/* input_artifacts/ || true + +# Run all PHP linux distribtests +# We run the distribtests even if some of the artifacts have failed to build, since that gives +# a better signal about which distribtest are affected by the currently broken artifact builds. +tools/run_tests/task_runner.py -f distribtest linux php -j 4 -x distribtests/sponge_log.xml || FAILED="true" + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/internal_ci/macos/grpc_distribtests_php.cfg b/tools/internal_ci/macos/grpc_distribtests_php.cfg new file mode 100644 index 00000000000..e06b917a164 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests_php.cfg @@ -0,0 +1,27 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_distribtests_php.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.*" + regex: "github/grpc/reports/**" + regex: "github/grpc/artifacts/**" + } +} diff --git a/tools/internal_ci/macos/grpc_distribtests_php.sh b/tools/internal_ci/macos/grpc_distribtests_php.sh new file mode 100644 index 00000000000..a6c33285d79 --- /dev/null +++ b/tools/internal_ci/macos/grpc_distribtests_php.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +export PREPARE_BUILD_INSTALL_DEPS_PHP=true +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +# Build all PHP macos artifacts +tools/run_tests/task_runner.py -f artifact macos php -j 4 -x build_artifacts/sponge_log.xml || FAILED="true" + +# PHP's "build_package" step is basically just a passthough, so the build artifacts can be used +# directly by the "distribtests" step (and we skip the "build_package" phase entirely on mac). + +# the next step expects to find the artifacts from the previous step in the "input_artifacts" folder. +# in addition to that, preserve the contents of "artifacts" directory since we want kokoro +# to upload its contents as job output artifacts +rm -rf input_artifacts +mkdir -p input_artifacts +cp -r artifacts/php_pecl_package_macos_*/* input_artifacts/ || true + +# Run all PHP linux distribtests +# We run the distribtests even if some of the artifacts have failed to build, since that gives +# a better signal about which distribtest are affected by the currently broken artifact builds. +tools/run_tests/task_runner.py -f distribtest macos php -j 4 -x distribtests/sponge_log.xml || FAILED="true" + +tools/internal_ci/helper_scripts/delete_nonartifacts.sh || true + +if [ "$FAILED" != "" ] +then + exit 1 +fi diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 0fa32cf3135..60157dd4124 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -275,10 +275,15 @@ class PHPArtifact: return [] def build_jobspec(self): - return create_docker_jobspec( - self.name, - 'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') + if self.platform == 'linux': + return create_docker_jobspec( + self.name, + 'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch), + 'tools/run_tests/artifacts/build_artifact_php.sh') + else: + return create_jobspec( + self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], + use_workspace=True) class ProtocArtifact: @@ -393,5 +398,6 @@ def targets(): PythonArtifact('windows', 'x64', 'Python39'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64') + PHPArtifact('linux', 'x64'), + PHPArtifact('macos', 'x64') ] diff --git a/tools/run_tests/artifacts/build_artifact_php.sh b/tools/run_tests/artifacts/build_artifact_php.sh index 629718e123b..dd242d1c258 100755 --- a/tools/run_tests/artifacts/build_artifact_php.sh +++ b/tools/run_tests/artifacts/build_artifact_php.sh @@ -21,12 +21,5 @@ mkdir -p "${ARTIFACTS_OUT}" # Build the PHP extension archive (this just zips all the files up) pear package -# Note: the extension compiled by this step is not being used in any -# way, i.e. they are not the pacakge being distributed. -# This is done here to get an early signal for compiling the PHP -# extension in some form. -find . -name "grpc-*.tgz" | cut -b3- | xargs pecl install -# Verified that the grpc extension is built properly. -php -d extension=grpc.so --re grpc | head -1 cp -r grpc-*.tgz "${ARTIFACTS_OUT}"/ diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index e263e09a357..46c3d5b8d94 100755 --- a/tools/run_tests/artifacts/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -20,4 +20,4 @@ cd "$(dirname "$0")/../../.." # All the PHP packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. mkdir -p artifacts/ -cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true +cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_pecl_package_linux_*/* artifacts/ || true diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 23798331553..370cff27297 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -238,7 +238,7 @@ class PHP7DistribTest(object): self.platform = platform self.arch = arch self.docker_suffix = docker_suffix - self.labels = ['distribtest', 'php7', platform, arch] + self.labels = ['distribtest', 'php', 'php7', platform, arch] if docker_suffix: self.labels.append(docker_suffix)