From b53e707c3c155171d746e99e7020f311f0af3d8b Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 23 May 2019 17:54:40 +0200 Subject: [PATCH 1/5] update bazel version in dockerfile --- templates/tools/dockerfile/bazel.include | 9 ++++++--- tools/dockerfile/test/bazel/Dockerfile | 9 ++++++--- tools/dockerfile/test/sanity/Dockerfile | 9 ++++++--- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/templates/tools/dockerfile/bazel.include b/templates/tools/dockerfile/bazel.include index 8888e938f43..3744dacc281 100644 --- a/templates/tools/dockerfile/bazel.include +++ b/templates/tools/dockerfile/bazel.include @@ -1,7 +1,10 @@ #======================== # Bazel installation +# Must be in sync with tools/bazel.sh +ENV BAZEL_VERSION 0.24.1 + RUN apt-get update && apt-get install -y wget && apt-get clean -RUN wget https://github.com/bazelbuild/bazel/releases/download/0.23.2/bazel-0.23.2-installer-linux-x86_64.sh && ${'\\'} - bash ./bazel-0.23.2-installer-linux-x86_64.sh && ${'\\'} - rm bazel-0.23.2-installer-linux-x86_64.sh +RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && ${'\\'} + bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && ${'\\'} + rm bazel-$BAZEL_VERSION-installer-linux-x86_64.sh diff --git a/tools/dockerfile/test/bazel/Dockerfile b/tools/dockerfile/test/bazel/Dockerfile index 2536fe299cb..b9fc409d939 100644 --- a/tools/dockerfile/test/bazel/Dockerfile +++ b/tools/dockerfile/test/bazel/Dockerfile @@ -51,10 +51,13 @@ RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.2.post1 six==1.10.0 t #======================== # Bazel installation +# Must be in sync with tools/bazel.sh +ENV BAZEL_VERSION 0.24.1 + RUN apt-get update && apt-get install -y wget && apt-get clean -RUN wget https://github.com/bazelbuild/bazel/releases/download/0.23.2/bazel-0.23.2-installer-linux-x86_64.sh && \ - bash ./bazel-0.23.2-installer-linux-x86_64.sh && \ - rm bazel-0.23.2-installer-linux-x86_64.sh +RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && \ + bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + rm bazel-$BAZEL_VERSION-installer-linux-x86_64.sh RUN mkdir -p /var/local/jenkins diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index 765bd7267a4..4ef4a0a9454 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -97,10 +97,13 @@ ENV CLANG_TIDY=clang-tidy #======================== # Bazel installation +# Must be in sync with tools/bazel.sh +ENV BAZEL_VERSION 0.24.1 + RUN apt-get update && apt-get install -y wget && apt-get clean -RUN wget https://github.com/bazelbuild/bazel/releases/download/0.23.2/bazel-0.23.2-installer-linux-x86_64.sh && \ - bash ./bazel-0.23.2-installer-linux-x86_64.sh && \ - rm bazel-0.23.2-installer-linux-x86_64.sh +RUN wget "https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh" && \ + bash ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + rm bazel-$BAZEL_VERSION-installer-linux-x86_64.sh # Define the default command. From 374ff3139a955b5df1fdb10b7a4cf228f597e1eb Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 23 May 2019 17:56:52 +0200 Subject: [PATCH 2/5] use bazel.sh for foundry RBE tests --- .../internal_ci/linux/grpc_bazel_on_foundry_base.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh index 93399f81e79..bcde79ecd8e 100755 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh @@ -15,26 +15,21 @@ set -ex -# Download bazel -temp_dir="$(mktemp -d)" -wget -q https://github.com/bazelbuild/bazel/releases/download/0.23.2/bazel-0.23.2-linux-x86_64 -O "${temp_dir}/bazel" -chmod 755 "${temp_dir}/bazel" -export PATH="${temp_dir}:${PATH}" -# This should show ${temp_dir}/bazel -which bazel - # change to grpc repo root cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc +# make sure bazel is available +tools/bazel.sh version + # to get "bazel" link for kokoro build, we need to generate # invocation UUID, set a flag for bazel to use it # and upload "bazel_invocation_ids" file as artifact. BAZEL_INVOCATION_ID="$(uuidgen)" echo "${BAZEL_INVOCATION_ID}" >"${KOKORO_ARTIFACTS_DIR}/bazel_invocation_ids" -bazel \ +tools/bazel.sh \ --bazelrc=tools/remote_build/kokoro.bazelrc \ test \ --invocation_id="${BAZEL_INVOCATION_ID}" \ From a3cc5ee574ca0e4c059ff353616ea307894aa223 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 23 May 2019 17:57:52 +0200 Subject: [PATCH 3/5] use bazel.sh in bazel RBE readme --- tools/remote_build/README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tools/remote_build/README.md b/tools/remote_build/README.md index 2cd5f03daf1..4cc3c7a0ea1 100644 --- a/tools/remote_build/README.md +++ b/tools/remote_build/README.md @@ -17,22 +17,27 @@ and tests run by Kokoro CI. ## Running remote build manually from dev workstation +*At the time being, tools/bazel.sh is used instead of invoking "bazel" directly +to overcome the bazel versioning problem (our BUILD files currently only work with +a specific window of bazel version and bazel.sh wrapper makes sure that version +is used).* + Run from repository root (opt, dbg): ``` # manual run of bazel tests remotely on Foundry -bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/... +tools/bazel.sh --bazelrc=tools/remote_build/manual.bazelrc test --config=opt //test/... ``` Sanitizer runs (asan, msan, tsan, ubsan): ``` # manual run of bazel tests remotely on Foundry with given sanitizer -bazel --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/... +tools/bazel.sh --bazelrc=tools/remote_build/manual.bazelrc test --config=asan //test/... ``` Run on Windows MSVC: ``` # RBE manual run only for c-core (must be run on a Windows host machine) -bazel --bazelrc=tools/remote_build/windows.bazelrc build :all [--credentials_json=(path to service account credentials)] +tools/bazel.sh --bazelrc=tools/remote_build/windows.bazelrc build :all [--credentials_json=(path to service account credentials)] ``` Available command line options can be found in From af279949d49109ee54d484b345abe5b78b8614ea Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 23 May 2019 18:16:03 +0200 Subject: [PATCH 4/5] update bazel_rbe.bat --- tools/internal_ci/windows/bazel_rbe.bat | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/internal_ci/windows/bazel_rbe.bat b/tools/internal_ci/windows/bazel_rbe.bat index 8f2c534c5ef..30c66896edf 100644 --- a/tools/internal_ci/windows/bazel_rbe.bat +++ b/tools/internal_ci/windows/bazel_rbe.bat @@ -13,7 +13,8 @@ @rem limitations under the License. @rem TODO(jtattermusch): make this generate less output -choco install bazel -y --version 0.23.2 --limit-output +@rem TODO(jtattermusch): use tools/bazel.sh script to keep the versions in sync +choco install bazel -y --version 0.24.1 --limit-output cd github/grpc set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% From 659f71099c163a4046c595460c119fa4cd16829d Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 29 May 2019 04:03:47 -0400 Subject: [PATCH 5/5] upgrade bazel toolchain to fix ubsan RBE build --- bazel/grpc_deps.bzl | 8 ++++---- tools/remote_build/rbe_common.bazelrc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index a4e6509782d..db24c7645b0 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -186,11 +186,11 @@ def grpc_deps(): if "bazel_toolchains" not in native.existing_rules(): http_archive( name = "bazel_toolchains", - sha256 = "67335b3563d9b67dc2550b8f27cc689b64fadac491e69ce78763d9ba894cc5cc", - strip_prefix = "bazel-toolchains-cddc376d428ada2927ad359211c3e356bd9c9fbb", + sha256 = "88e818f9f03628eef609c8429c210ecf265ffe46c2af095f36c7ef8b1855fef5", + strip_prefix = "bazel-toolchains-92dd8a7a518a2fb7ba992d47c8b38299fe0be825", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz", - "https://github.com/bazelbuild/bazel-toolchains/archive/cddc376d428ada2927ad359211c3e356bd9c9fbb.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz", + "https://github.com/bazelbuild/bazel-toolchains/archive/92dd8a7a518a2fb7ba992d47c8b38299fe0be825.tar.gz", ], ) diff --git a/tools/remote_build/rbe_common.bazelrc b/tools/remote_build/rbe_common.bazelrc index 69394fe092b..63b49c5c42d 100644 --- a/tools/remote_build/rbe_common.bazelrc +++ b/tools/remote_build/rbe_common.bazelrc @@ -84,4 +84,4 @@ build:ubsan --copt=-gmlt # TODO(jtattermusch): use more reasonable test timeout build:ubsan --test_timeout=3600 # override the config-agnostic crosstool_top -build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.21.0/ubsan:toolchain +build:ubsan --crosstool_top=@bazel_toolchains//configs/experimental/ubuntu16_04_clang/1.2/bazel_0.23.0/ubsan:toolchain