mirror of https://github.com/grpc/grpc.git
[bazelified tests] First attempt at bazelified rules for building artifacts/packages (#34462)
Foundation for being able to bazelify the build artifact -> build package -> distribtest workflow tests. Main ideas: - "build artifact" and "build packages" will be represented by a custom genrule (that runs the build on RBE under a docker container). - since genrule doesn't support displaying logs for each target as a separate "target log" (in the same way that bazel tests do), and we generally want readable per-target logs for the bazelified test, a pair of targets will be created for each "build artifact task": - a genrule that actually performs the build, creates an archive with artifacts and stores the exitcode and build log as rule outputs - a corresponding "build_test" sh_test that simply looks at the result of the genrule and presents the build log and build result a "target log" for this test.pull/34589/head
Jan Tattermusch
1 year ago
committed by
GitHub
parent
b581a24a4c
commit
3b40af376e
15 changed files with 786 additions and 1 deletions
@ -0,0 +1,65 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -e |
||||
|
||||
ARCHIVE_WITH_SUBMODULES="$1" |
||||
BUILD_SCRIPT="$2" |
||||
EXIT_CODE_FILE="$3" |
||||
SCRIPT_LOG_FILE="$4" |
||||
ARTIFACTS_OUT_FILE="$5" |
||||
shift 5 |
||||
|
||||
# Extract grpc repo archive |
||||
tar -xopf ${ARCHIVE_WITH_SUBMODULES} |
||||
cd grpc |
||||
|
||||
# Extract all input archives with artifacts into input_artifacts directory |
||||
# TODO(jtattermusch): Deduplicate the snippet below (it appears in multiple files). |
||||
mkdir -p input_artifacts |
||||
pushd input_artifacts >/dev/null |
||||
# all remaining args are .tar.gz archives with input artifacts |
||||
for input_artifact_archive in "$@" |
||||
do |
||||
# extract the .tar.gz with artifacts into a directory named after a basename |
||||
# of the archive itself (and strip the "artifact/" prefix) |
||||
# Note that input artifacts from different dependencies can have files |
||||
# with the same name, so disambiguating through the name of the archive |
||||
# is important. |
||||
archive_extract_dir="$(basename ${input_artifact_archive} .tar.gz)" |
||||
mkdir -p "${archive_extract_dir}" |
||||
pushd "${archive_extract_dir}" >/dev/null |
||||
tar --strip-components=1 -xopf ../../../${input_artifact_archive} |
||||
popd >/dev/null |
||||
done |
||||
popd >/dev/null |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
# Run the build script with args, storing its stdout and stderr |
||||
# in a log file. |
||||
SCRIPT_EXIT_CODE=0 |
||||
../"${BUILD_SCRIPT}" >"../${SCRIPT_LOG_FILE}" 2>&1 || SCRIPT_EXIT_CODE="$?" |
||||
|
||||
# Store build script's exitcode in a file. |
||||
# Note that the build atifacts task will terminate with success even when |
||||
# there was an error building the artifacts. |
||||
# The error status (an associated log) will be reported by an associated |
||||
# bazel test. |
||||
echo "${SCRIPT_EXIT_CODE}" >"../${EXIT_CODE_FILE}" |
||||
|
||||
# collect the artifacts |
||||
# TODO(jtattermusch): add tar flags to create deterministic tar archive |
||||
tar -czvf ../"${ARTIFACTS_OUT_FILE}" artifacts |
||||
@ -0,0 +1,55 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -e |
||||
|
||||
EXIT_CODE_FILE="$1" |
||||
SCRIPT_LOG_FILE="$2" |
||||
ARTIFACTS_ARCHIVE="$3" |
||||
shift 3 |
||||
|
||||
BUILD_ARTIFACT_EXITCODE="$(cat ${EXIT_CODE_FILE})" |
||||
|
||||
echo "Build artifact/package task for '${ARTIFACTS_ARCHIVE}' has finished with exitcode ${BUILD_ARTIFACT_EXITCODE}." |
||||
|
||||
echo "BUILD LOG" |
||||
echo "--------------" |
||||
cat "${SCRIPT_LOG_FILE}" |
||||
echo "--------------" |
||||
echo |
||||
|
||||
# Try extracting the archive with artifacts (and list the files) |
||||
mkdir -p input_artifacts |
||||
pushd input_artifacts >/dev/null |
||||
echo "Artifacts that were built by the build artifact/package task:" |
||||
echo "--------------" |
||||
# TODO(jtattermusch): strip top level artifacts/ directory from the archive? |
||||
tar -xopvf ../${ARTIFACTS_ARCHIVE} |
||||
echo "--------------" |
||||
popd >/dev/null |
||||
|
||||
# Add artifact archive to the "undeclared test outputs" directory |
||||
# to make it readily available in the resultstore UI. |
||||
# See bazel docs for TEST_UNDECLARED_OUTPUTS_DIR. |
||||
mkdir -p "${TEST_UNDECLARED_OUTPUTS_DIR}" |
||||
cp "${ARTIFACTS_ARCHIVE}" "${TEST_UNDECLARED_OUTPUTS_DIR}" || true |
||||
|
||||
if [ "${BUILD_ARTIFACT_EXITCODE}" -eq "0" ] |
||||
then |
||||
echo "SUCCESS: Build artifact/package task for '${ARTIFACTS_ARCHIVE}' ran successfully." |
||||
else |
||||
echo "FAIL: Build artifact/package task for '${ARTIFACTS_ARCHIVE}' failed with exitcode ${BUILD_ARTIFACT_EXITCODE}." |
||||
exit 1 |
||||
fi |
||||
@ -0,0 +1,49 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -e |
||||
|
||||
ARCHIVE_WITH_SUBMODULES="$1" |
||||
BUILD_SCRIPT="$2" |
||||
shift 2 |
||||
|
||||
# Extract grpc repo archive |
||||
tar -xopf ${ARCHIVE_WITH_SUBMODULES} |
||||
cd grpc |
||||
|
||||
# Extract all input archives with artifacts into input_artifacts directory |
||||
# TODO(jtattermusch): Deduplicate the snippet below (it appears in multiple files). |
||||
mkdir -p input_artifacts |
||||
pushd input_artifacts >/dev/null |
||||
# all remaining args are .tar.gz archives with input artifacts |
||||
for input_artifact_archive in "$@" |
||||
do |
||||
# extract the .tar.gz with artifacts into a directory named after a basename |
||||
# of the archive itself (and strip the "artifact/" prefix) |
||||
# Note that input artifacts from different dependencies can have files |
||||
# with the same name, so disambiguating through the name of the archive |
||||
# is important. |
||||
archive_extract_dir="$(basename ${input_artifact_archive} .tar.gz)" |
||||
mkdir -p "${archive_extract_dir}" |
||||
pushd "${archive_extract_dir}" >/dev/null |
||||
tar --strip-components=1 -xopf ../../../${input_artifact_archive} |
||||
popd >/dev/null |
||||
done |
||||
popd >/dev/null |
||||
|
||||
ls -lR input_artifacts |
||||
|
||||
# Run build script passed as arg. |
||||
"${BUILD_SCRIPT}" |
||||
@ -0,0 +1,20 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_php.sh |
||||
@ -0,0 +1,25 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# compile/link options extracted from ProtocArtifact in tools/run_tests/artifacts/artifact_targets.py |
||||
export LDFLAGS="${LDFLAGS} -static-libgcc -static-libstdc++ -s" |
||||
# set build parallelism to fit the machine configuration of bazelified tests RBE pool. |
||||
export GRPC_PROTOC_BUILD_COMPILER_JOBS=8 |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_protoc.sh |
||||
@ -0,0 +1,36 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# env variable values extracted from PythonArtifact in tools/run_tests/artifacts/artifact_targets.py |
||||
# TODO(jtattermusch): find a better way of configuring the python artifact build (the current approach mostly serves as a demonstration) |
||||
export PYTHON=/opt/python/cp311-cp311/bin/python |
||||
export PIP=/opt/python/cp311-cp311/bin/pip |
||||
export GRPC_SKIP_PIP_CYTHON_UPGRADE=TRUE |
||||
export GRPC_RUN_AUDITWHEEL_REPAIR=TRUE |
||||
export GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS=TRUE |
||||
|
||||
# Without this python cannot find the c++ compiler |
||||
# TODO(jtattermusch): find better solution to prevent bazel from |
||||
# restricting path contents |
||||
export PATH="/opt/rh/devtoolset-10/root/usr/bin:$PATH" |
||||
|
||||
# set build parallelism to fit the machine configuration of bazelified tests RBE pool. |
||||
export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=8 |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_python.sh |
||||
@ -0,0 +1,36 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# env variable values extracted from PythonArtifact in tools/run_tests/artifacts/artifact_targets.py |
||||
# TODO(jtattermusch): find a better way of configuring the python artifact build (the current approach mostly serves as a demonstration) |
||||
export PYTHON=/opt/python/cp37-cp37m/bin/python |
||||
export PIP=/opt/python/cp37-cp37m/bin/pip |
||||
export GRPC_SKIP_PIP_CYTHON_UPGRADE=TRUE |
||||
export GRPC_RUN_AUDITWHEEL_REPAIR=TRUE |
||||
export GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS=TRUE |
||||
|
||||
# Without this python cannot find the c++ compiler |
||||
# TODO(jtattermusch): find better solution to prevent bazel from |
||||
# restricting path contents |
||||
export PATH="/opt/rh/devtoolset-10/root/usr/bin:$PATH" |
||||
|
||||
# set build parallelism to fit the machine configuration of bazelified tests RBE pool. |
||||
export GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS=8 |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
ARTIFACTS_OUT=artifacts tools/run_tests/artifacts/build_artifact_python.sh |
||||
@ -0,0 +1,43 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
# List all input artifacts we obtained for easier troubleshooting. |
||||
ls -lR input_artifacts |
||||
|
||||
# Put the input artifacts where the legacy logic for building |
||||
# C# package expects to find them. |
||||
# See artifact_targets.py and package_targets.py for details. |
||||
# TODO(jtattermusch): get rid of the manual renames of artifact directories. |
||||
export EXTERNAL_GIT_ROOT="$(pwd)" |
||||
mv input_artifacts/artifact_protoc_linux_aarch64 input_artifacts/protoc_linux_aarch64 || true |
||||
mv input_artifacts/artifact_protoc_linux_x64 input_artifacts/protoc_linux_x64 || true |
||||
mv input_artifacts/artifact_protoc_linux_x86 input_artifacts/protoc_linux_x86 || true |
||||
|
||||
# In the bazel workflow, we only have linux protoc artifact at hand, |
||||
# so we can only build a "singleplatform" version of the C# package. |
||||
export GRPC_CSHARP_BUILD_SINGLE_PLATFORM_NUGET=1 |
||||
|
||||
# TODO(jtattermusch): when building the C# nugets, the current git commit SHA |
||||
# is retrieved and stored as package metadata. But when running |
||||
# as bazelified test, this is not possible since we're not in a git |
||||
# workspace when running the build. This is ok for testing purposes |
||||
# but would be a problem if building a production package |
||||
# for the end users. |
||||
|
||||
src/csharp/build_nuget.sh |
||||
@ -0,0 +1,27 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
mkdir -p artifacts |
||||
|
||||
# List all input artifacts we obtained for easier troubleshooting. |
||||
ls -lR input_artifacts |
||||
|
||||
# All the python packages have been built in the artifact phase already |
||||
# and we only collect them here to deliver them to the distribtest phase. |
||||
# This is the same logic as in "tools/run_tests/artifacts/build_package_python.sh", |
||||
# but expects different layout under input_artifacts. |
||||
cp -r input_artifacts/artifact_python_*/* artifacts/ || true |
||||
@ -0,0 +1,28 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# List all input artifacts we obtained for easier troubleshooting. |
||||
ls -lR input_artifacts |
||||
|
||||
# Put the input packages where the legacy logic for running |
||||
# C# distribtest expects to find them. |
||||
# See distribtest_targets.py for details. |
||||
# TODO(jtattermusch): get rid of the manual renames of artifact files. |
||||
export EXTERNAL_GIT_ROOT="$(pwd)" |
||||
mv input_artifacts/package_csharp_linux/* input_artifacts/ || true |
||||
|
||||
test/distrib/csharp/run_distrib_test_dotnetcli.sh |
||||
@ -0,0 +1,28 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# List all input artifacts we obtained for easier troubleshooting. |
||||
ls -lR input_artifacts |
||||
|
||||
# Put the input packages where the legacy logic for running |
||||
# PHP distribtest expects to find them. |
||||
# See distribtest_targets.py for details. |
||||
# TODO(jtattermusch): get rid of the manual renames of artifact files. |
||||
export EXTERNAL_GIT_ROOT="$(pwd)" |
||||
mv input_artifacts/artifact_php_linux_x64/* input_artifacts/ || true |
||||
|
||||
test/distrib/php/run_distrib_test.sh |
||||
@ -0,0 +1,28 @@ |
||||
#!/bin/bash |
||||
# Copyright 2023 The gRPC Authors |
||||
# |
||||
# Licensed under the Apache License, Version 2.0 (the "License"); |
||||
# you may not use this file except in compliance with the License. |
||||
# You may obtain a copy of the License at |
||||
# |
||||
# http://www.apache.org/licenses/LICENSE-2.0 |
||||
# |
||||
# Unless required by applicable law or agreed to in writing, software |
||||
# distributed under the License is distributed on an "AS IS" BASIS, |
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
||||
# See the License for the specific language governing permissions and |
||||
# limitations under the License. |
||||
|
||||
set -ex |
||||
|
||||
# List all input artifacts we obtained for easier troubleshooting. |
||||
ls -lR input_artifacts |
||||
|
||||
# Put the input packages where the legacy logic for running |
||||
# Python distribtest expects to find them. |
||||
# See distribtest_targets.py for details. |
||||
# TODO(jtattermusch): get rid of the manual renames of artifact files. |
||||
export EXTERNAL_GIT_ROOT="$(pwd)" |
||||
mv input_artifacts/package_python_linux/* input_artifacts/ || true |
||||
|
||||
test/distrib/python/run_binary_distrib_test.sh |
||||
Loading…
Reference in new issue