From a64ddff19c4ca8b5c2c9ae1aa41b70830a20eaed Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 5 Jun 2019 14:16:51 -0400 Subject: [PATCH 1/4] hijack bazel invocations in grpc workspace --- tools/{bazel.sh => bazel} | 26 +++++++++++++++----------- tools/remote_build/README.md | 11 +++-------- 2 files changed, 18 insertions(+), 19 deletions(-) rename tools/{bazel.sh => bazel} (65%) diff --git a/tools/bazel.sh b/tools/bazel similarity index 65% rename from tools/bazel.sh rename to tools/bazel index 37fd2a242bf..15c78dd0ffa 100755 --- a/tools/bazel.sh +++ b/tools/bazel @@ -19,17 +19,21 @@ # supported version, and then calling it. This way, we can make sure # that running bazel will always get meaningful results, at least # until Bazel 1.0 is released. +# NOTE: This script relies on bazel's feature where //tools/bazel +# script can be used to hijack "bazel" invocations in given workspace. set -e VERSION=0.24.1 -CWD=`pwd` +echo "INFO: Running bazel wrapper (see //tools/bazel for details), bazel version $VERSION will be used instead of system-wide bazel installation." + +CWD=$(pwd) BASEURL=https://github.com/bazelbuild/bazel/releases/download/ -cd `dirname $0` -TOOLDIR=`pwd` +cd "$(dirname "$0")" +TOOLDIR=$(pwd) -case `uname -sm` in +case $(uname -sm) in "Linux x86_64") suffix=linux-x86_64 ;; @@ -37,17 +41,17 @@ case `uname -sm` in suffix=darwin-x86_64 ;; *) - echo "Unsupported architecture: `uname -sm`" + echo "Unsupported architecture: $(uname -sm)" exit 1 ;; esac -filename=bazel-$VERSION-$suffix +filename="bazel-$VERSION-$suffix" -if [ ! -x $filename ] ; then - curl -L $BASEURL/$VERSION/$filename > $filename - chmod a+x $filename +if [ ! -x "$filename" ] ; then + curl -L "$BASEURL/$VERSION/$filename" > "$filename" + chmod a+x "$filename" fi -cd $CWD -$TOOLDIR/$filename $@ +cd "$CWD" +"$TOOLDIR/$filename" $@ diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md index 4cc3c7a0ea1..2cd5f03daf1 100644 --- a/tools/remote_build/README.md +++ b/tools/remote_build/README.md @@ -17,27 +17,22 @@ and tests run by Kokoro CI. ## Running remote build manually from dev workstation -*At the time being, tools/bazel.sh is used instead of invoking "bazel" directly -to overcome the bazel versioning problem (our BUILD files currently only work with -a specific window of bazel version and bazel.sh wrapper makes sure that version -is used).* - Run from repository root (opt, dbg): ``` # manual run of bazel tests remotely on Foundry -tools/bazel.sh --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/... +bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/... ``` Sanitizer runs (asan, msan, tsan, ubsan): ``` # manual run of bazel tests remotely on Foundry with given sanitizer -tools/bazel.sh --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/... +bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/... ``` Run on Windows MSVC: ``` # RBE manual run only for c-core (must be run on a Windows host machine) -tools/bazel.sh --bazelrc=tools/remote_build/windows.bazelrc build :all [--credentials_json=(path to service account credentials)] +bazel --bazelrc=tools/remote_build/windows.bazelrc build :all [--credentials_json=(path to service account credentials)] ``` Available command line options can be found in From ed7e8c53e0a4b3a10b65919e059999e00c12a0bb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 5 Jun 2019 14:21:51 -0400 Subject: [PATCH 2/4] update scripts --- templates/tools/dockerfile/bazel.include | 2 +- tools/dockerfile/test/bazel/Dockerfile | 2 +- tools/dockerfile/test/sanity/Dockerfile | 2 +- tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh | 4 ++-- tools/internal_ci/windows/bazel_rbe.bat | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/templates/tools/dockerfile/bazel.include b/templates/tools/dockerfile/bazel.include index 3744dacc281..b46845e218d 100644 --- a/templates/tools/dockerfile/bazel.include +++ b/templates/tools/dockerfile/bazel.include @@ -1,7 +1,7 @@ #======================== # Bazel installation -# Must be in sync with tools/bazel.sh +# Must be in sync with tools/bazel ENV BAZEL_VERSION 0.24.1 RUN apt-get update && apt-get install -y wget && apt-get clean diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index b9fc409d939..0e958542f9a 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -51,7 +51,7 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t #======================== # Bazel installation -# Must be in sync with tools/bazel.sh +# Must be in sync with tools/bazel ENV BAZEL_VERSION 0.24.1 RUN apt-get update && apt-get install -y wget && apt-get clean diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 4ef4a0a9454..18aa89d8bf9 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -97,7 +97,7 @@ ENV CLANG_TIDY=clang-tidy #======================== # Bazel installation -# Must be in sync with tools/bazel.sh +# Must be in sync with tools/bazel ENV BAZEL_VERSION 0.24.1 RUN apt-get update && apt-get install -y wget && apt-get clean diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index bcde79ecd8e..5a358acc09d 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -21,7 +21,7 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc # make sure bazel is available -tools/bazel.sh version +tools/bazel version # to get "bazel" link for kokoro build, we need to generate # invocation UUID, set a flag for bazel to use it @@ -29,7 +29,7 @@ tools/bazel.sh version BAZEL_INVOCATION_ID="$(uuidgen)" echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" -tools/bazel.sh \ +tools/bazel \ --bazelrc=tools/remote_build/kokoro.bazelrc \ test \ --invocation_id="${BAZEL_INVOCATION_ID}" \ diff --git a/tools/internal_ci/windows/bazel_rbe.bat b/tools/internal_ci/windows/bazel_rbe.bat index 30c66896edf..8abab687ed2 100644 --- a/tools/internal_ci/windows/bazel_rbe.bat +++ b/tools/internal_ci/windows/bazel_rbe.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem TODO(jtattermusch): make this generate less output -@rem TODO(jtattermusch): use tools/bazel.sh script to keep the versions in sync +@rem TODO(jtattermusch): use tools/bazel script to keep the versions in sync choco install bazel -y --version 0.24.1 --limit-output cd github/grpc From f13727dfd110b7e1c95d24acf607d08ca44c4a35 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 5 Jun 2019 16:45:41 -0400 Subject: [PATCH 3/4] add option to disable bazel wrapper --- tools/bazel | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/bazel b/tools/bazel index 15c78dd0ffa..bacfeaddfd2 100755 --- a/tools/bazel +++ b/tools/bazel @@ -24,13 +24,20 @@ set -e +# First of all, if DISABLE_BAZEL_WRAPPER is set, just use BAZEL_REAL as set by +# https://github.com/bazelbuild/bazel/blob/master/scripts/packages/bazel.sh +# that originally invoked this script. +if [ "${BAZEL_REAL}" != "" ] && [ "${DISABLE_BAZEL_WRAPPER}" != "" ] +then + exec -a "$0" "${BAZEL_REAL}" "$@" +fi + VERSION=0.24.1 echo "INFO: Running bazel wrapper (see //tools/bazel for details), bazel version $VERSION will be used instead of system-wide bazel installation." -CWD=$(pwd) BASEURL=https://github.com/bazelbuild/bazel/releases/download/ -cd "$(dirname "$0")" +pushd "$(dirname "$0")" >/dev/null TOOLDIR=$(pwd) case $(uname -sm) in @@ -53,5 +60,5 @@ if [ ! -x "$filename" ] ; then chmod a+x "$filename" fi -cd "$CWD" -"$TOOLDIR/$filename" $@ +popd >/dev/null +exec "$TOOLDIR/$filename" "$@" From e66fb82495da076607eb27cc940714748d870762 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 5 Jun 2019 16:53:12 -0400 Subject: [PATCH 4/4] disable bazel wrapper inside docker containers --- templates/tools/dockerfile/bazel.include | 3 +++ tools/dockerfile/test/bazel/Dockerfile | 3 +++ tools/dockerfile/test/sanity/Dockerfile | 3 +++ 3 files changed, 9 insertions(+) diff --git a/templates/tools/dockerfile/bazel.include b/templates/tools/dockerfile/bazel.include index b46845e218d..272936007da 100644 --- a/templates/tools/dockerfile/bazel.include +++ b/templates/tools/dockerfile/bazel.include @@ -4,6 +4,9 @@ # Must be in sync with tools/bazel ENV BAZEL_VERSION 0.24.1 +# The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. +ENV DISABLE_BAZEL_WRAPPER 1 + RUN apt-get update && apt-get install -y wget && apt-get clean RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && ${'\\'} bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && ${'\\'} diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index 0e958542f9a..df7a7a2f32a 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -54,6 +54,9 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t # Must be in sync with tools/bazel ENV BAZEL_VERSION 0.24.1 +# The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. +ENV DISABLE_BAZEL_WRAPPER 1 + RUN apt-get update && apt-get install -y wget && apt-get clean RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && \ bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 18aa89d8bf9..9a080d0efb2 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -100,6 +100,9 @@ ENV CLANG_TIDY=clang-tidy # Must be in sync with tools/bazel ENV BAZEL_VERSION 0.24.1 +# The correct bazel version is already preinstalled, no need to use //tools/bazel wrapper. +ENV DISABLE_BAZEL_WRAPPER 1 + RUN apt-get update && apt-get install -y wget && apt-get clean RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && \ bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \