From 6816da50cf06a135f145a3cef3b5bd45ae78ef43 Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Thu, 2 Feb 2023 11:48:37 -0800 Subject: [PATCH] Migrate non-Linux C++ tests to GHA actions PiperOrigin-RevId: 506683178 --- .github/actions/bazel-docker/action.yml | 38 ++++------ .github/actions/bazel/action.yml | 75 +++++++++++++++++++ .../{non-bazel-docker => docker}/action.yml | 3 + .../actions/internal/bazel-setup/action.yml | 59 +++++++++++++++ .../actions/internal/docker-run/action.yml | 4 + .../{docker-auth => gcloud-auth}/action.yml | 7 +- .../actions/internal/setup-runner/action.yml | 17 +++++ .github/workflows/test_cpp.yml | 27 ++++++- .github/workflows/test_php.yml | 2 +- ci/Linux.bazelrc | 3 + ci/Windows.bazelrc | 2 + ci/common.bazelrc | 33 ++++++++ ci/macOS.bazelrc | 3 + .../protobuf/io/zero_copy_stream_unittest.cc | 6 +- 14 files changed, 246 insertions(+), 33 deletions(-) create mode 100644 .github/actions/bazel/action.yml rename .github/actions/{non-bazel-docker => docker}/action.yml (93%) create mode 100644 .github/actions/internal/bazel-setup/action.yml rename .github/actions/internal/{docker-auth => gcloud-auth}/action.yml (76%) create mode 100644 .github/actions/internal/setup-runner/action.yml create mode 100644 ci/Linux.bazelrc create mode 100644 ci/Windows.bazelrc create mode 100644 ci/common.bazelrc create mode 100644 ci/macOS.bazelrc diff --git a/.github/actions/bazel-docker/action.yml b/.github/actions/bazel-docker/action.yml index 407757447e..c42916357a 100644 --- a/.github/actions/bazel-docker/action.yml +++ b/.github/actions/bazel-docker/action.yml @@ -11,7 +11,7 @@ inputs: description: "The docker image to use" type: string bazel-cache: - required: false + required: true description: > A unique path for the Bazel cache. This will trigger the generation of a BAZEL_CACHE environment variable inside the container that provides @@ -31,34 +31,24 @@ runs: steps: - name: Authenticate id: auth - uses: ./.github/actions/internal/docker-auth + uses: ./.github/actions/internal/gcloud-auth with: credentials: ${{ inputs.credentials }} - - name: Validate inputs - if: ${{ inputs.bash && inputs.bazel}} - shell: bash - run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 + - name: Setup Runner + uses: ./.github/actions/internal/setup-runner - - name: Initialize Bazel flags - shell: bash - run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV - - - name: Configure Bazel caching - shell: bash - # Skip bazel cache for local act runs due to issue with credential files - # and nested docker images - if: ${{ inputs.bazel-cache && !github.event.act_local_test }} - run: > - echo "BAZEL_FLAGS=$BAZEL_FLAGS - --google_credentials=/workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) - --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV + - name: Setup Bazel + id: bazel + uses: ./.github/actions/internal/bazel-setup + with: + credentials-file: /workspace/$(basename ${{ steps.auth.outputs.credentials-file }}) + bazel-cache: ${{ inputs.bazel-cache }} - - name: Configure Bazel cache writing + - name: Validate inputs + if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} shell: bash - # External PRs should never write to our caches. - if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} - run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV + run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 - name: Run Bash Docker uses: ./.github/actions/internal/docker-run @@ -73,4 +63,4 @@ runs: if: ${{ !inputs.bash }} with: image: ${{ inputs.image }} - command: ${{ inputs.bazel }} $BAZEL_FLAGS + command: ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }} diff --git a/.github/actions/bazel/action.yml b/.github/actions/bazel/action.yml new file mode 100644 index 0000000000..8501615eed --- /dev/null +++ b/.github/actions/bazel/action.yml @@ -0,0 +1,75 @@ +name: 'Docker Bazel Run' +description: 'Run a Bazel-based docker image for Protobuf CI testing' +inputs: + credentials: + required: true + description: The GCP credentials to use for reading the docker image + type: string + bazel-cache: + required: true + description: > + A unique path for the Bazel cache. This will trigger the generation + of a BAZEL_CACHE environment variable inside the container that provides + the appropriate flags for any bazel command. + type: string + version: + required: false + description: A pinned Bazel version to use + default: '5.1.1' + type: string + bazel: + required: false + description: The Bazel command to run + type: string + bash: + required: false + description: > + A bash command to run. $BAZEL_FLAGS and $BAZEL_STARTUP_FLAGS will be + available to use for bazel runs. + type: string + +runs: + using: 'composite' + steps: + - name: Authenticate + id: auth + uses: ./.github/actions/internal/gcloud-auth + with: + credentials: ${{ inputs.credentials }} + + - name: Setup Runner + uses: ./.github/actions/internal/setup-runner + + - name: Setup Bazel + id: bazel + uses: ./.github/actions/internal/bazel-setup + with: + credentials-file: ${{ steps.auth.outputs.credentials-file }} + bazel-cache: ${{ inputs.bazel-cache }} + + - name: Validate inputs + if: ${{ (inputs.bash && inputs.bazel) || (!inputs.bash && !inputs.bazel) }} + shell: bash + run: echo "Invalid specification of both non-Bazel and Bazel command"; exit 1 + + - name: Output Bazel version + env: + USE_BAZEL_VERSION: ${{ inputs.version }} + shell: bash + run: bazelisk version + + - name: Run Bash + env: + USE_BAZEL_VERSION: ${{ inputs.version }} + if: ${{ inputs.bash }} + run: ${{ inputs.bash }} + shell: bash + + - name: Run Bazel + env: + USE_BAZEL_VERSION: ${{ inputs.version }} + if: ${{ !inputs.bash }} + run: > + bazelisk ${{ steps.bazel.outputs.bazel-startup-flags }} + ${{ inputs.bazel }} ${{ steps.bazel.outputs.bazel-flags }} + shell: bash diff --git a/.github/actions/non-bazel-docker/action.yml b/.github/actions/docker/action.yml similarity index 93% rename from .github/actions/non-bazel-docker/action.yml rename to .github/actions/docker/action.yml index 23e502a653..93bab6be70 100644 --- a/.github/actions/non-bazel-docker/action.yml +++ b/.github/actions/docker/action.yml @@ -17,6 +17,9 @@ inputs: runs: using: 'composite' steps: + - name: Setup Runner + uses: ./.github/actions/internal/setup-runner + - name: Update stale files using Bazel uses: ./.github/actions/bazel-docker with: diff --git a/.github/actions/internal/bazel-setup/action.yml b/.github/actions/internal/bazel-setup/action.yml new file mode 100644 index 0000000000..28d916a3de --- /dev/null +++ b/.github/actions/internal/bazel-setup/action.yml @@ -0,0 +1,59 @@ +name: Setup Bazel +description: Setup a Bazel environment for Protobuf CI testing +inputs: + credentials-file: + required: true + description: The GCP credentials file to use for caching + type: string + bazel-cache: + required: true + description: A unique path for the Bazel cache. + type: string + +outputs: + bazel-flags: + description: Bazel flags that should be sent to all Bazel invocations + value: ${{ steps.output.outputs.bazel-flags }} + bazel-startup-flags: + description: Bazel startup flags that should be sent to all Bazel invocations + value: ${{ steps.output.outputs.bazel-startup-flags }} + +runs: + using: 'composite' + steps: + - name: Initialize Windows startup flags + if: runner.os == 'Windows' + shell: bash + run: echo "BAZEL_STARTUP_FLAGS=--output_user_root=C:/tmp --windows_enable_symlinks" >> $GITHUB_ENV + + - name: Initialize Bazel flags + shell: bash + run: echo "BAZEL_FLAGS=--keep_going --test_output=errors --test_timeout=600" >> $GITHUB_ENV + + - name: Initialize Windows-specific Bazel flags + if: runner.os == 'Windows' + shell: bash + run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --enable_runfiles" >> $GITHUB_ENV + + - name: Configure Bazel caching + # Skip bazel cache for local act runs due to issue with credential files + # and nested docker images + if: ${{ inputs.bazel-cache && !github.event.act_local_test }} + shell: bash + run: > + echo "BAZEL_FLAGS=$BAZEL_FLAGS + --google_credentials='${{ inputs.credentials-file }}' + --remote_cache=https://storage.googleapis.com/protobuf-bazel-cache/protobuf/gha/${{ inputs.bazel-cache }}" >> $GITHUB_ENV + + - name: Configure Bazel cache writing + # External runs should never write to our caches. + if: ${{ github.event_name != 'pull_request_target' && inputs.bazel-cache && !github.event.act_local_test }} + shell: bash + run: echo "BAZEL_FLAGS=$BAZEL_FLAGS --remote_upload_local_results" >> $GITHUB_ENV + + - name: Output Bazel flags + id: output + shell: bash + run: | + echo "bazel-flags=$BAZEL_FLAGS" >> $GITHUB_OUTPUT + echo "bazel-startup-flags=$BAZEL_STARTUP_FLAGS" >> $GITHUB_OUTPUT diff --git a/.github/actions/internal/docker-run/action.yml b/.github/actions/internal/docker-run/action.yml index 9e29418466..1cc14894d8 100644 --- a/.github/actions/internal/docker-run/action.yml +++ b/.github/actions/internal/docker-run/action.yml @@ -22,6 +22,10 @@ inputs: runs: using: 'composite' steps: + - name: Authenticate for GAR use + shell: bash + run: gcloud auth configure-docker -q us-docker.pkg.dev + - name: Setup QEMU for possible emulation uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0 diff --git a/.github/actions/internal/docker-auth/action.yml b/.github/actions/internal/gcloud-auth/action.yml similarity index 76% rename from .github/actions/internal/docker-auth/action.yml rename to .github/actions/internal/gcloud-auth/action.yml index 482ce7a0e8..8c0f2ca0a3 100644 --- a/.github/actions/internal/docker-auth/action.yml +++ b/.github/actions/internal/gcloud-auth/action.yml @@ -1,9 +1,9 @@ -name: 'Authenticate Docker' +name: 'Authenticate for GCP' description: 'Authenticate a workflow for Protobuf CI testing' inputs: credentials: required: true - description: "The GCP credentials to use for reading the docker image" + description: "The GCP credentials to use for GCP" type: string outputs: @@ -24,6 +24,3 @@ runs: - name: Use gcloud CLI shell: bash run: gcloud info - - name: Authenticate for GAR use - shell: bash - run: gcloud auth configure-docker -q us-docker.pkg.dev diff --git a/.github/actions/internal/setup-runner/action.yml b/.github/actions/internal/setup-runner/action.yml new file mode 100644 index 0000000000..5c4f783e6a --- /dev/null +++ b/.github/actions/internal/setup-runner/action.yml @@ -0,0 +1,17 @@ +name: Setup CI Runner +# TODO(b/267357823) Consider moving this to it's own repository and include +# a call to actions/checkout. +description: Setup any platform-specific adjustments we need to make for CI +runs: + using: 'composite' + steps: + - name: Fix Windows line breaks + if: runner.os == 'Windows' + shell: bash + run: find . -type f -print0 | xargs -0 d2u 2>/dev/null + + - name: Install bazelrc files + shell: bash + run: | + cp ci/*.bazelrc . + cp -f ${{ runner.os }}.bazelrc .bazelrc diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 5909f9d485..03d3cb3c2b 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -88,8 +88,33 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/non-bazel-docker + uses: ./.github/actions/docker with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: ${{ matrix.command }} + + non-linux: + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + include: + - name: MacOS + os: macos-12 + bazel: //src/... + - name: Windows + os: windows-2019 + bazel: //src/... @com_google_protobuf_examples//... --test_tag_filters=-conformance --build_tag_filters=-conformance + name: ${{ matrix.name }} Bazel + runs-on: ${{ matrix.os }} + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + ref: ${{ inputs.safe-checkout }} + - name: Run tests + uses: ./.github/actions/bazel + with: + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + bazel: test ${{ matrix.bazel }} + bazel-cache: cpp_${{ matrix.os }} diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index 0bf0cebdc8..7023bf5689 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -43,7 +43,7 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} - name: Run tests - uses: ./.github/actions/non-bazel-docker + uses: ./.github/actions/docker with: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/php:${{ matrix.version }}-6e95c0e221e4bd52e3b4dc1398c6336985196931 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} diff --git a/ci/Linux.bazelrc b/ci/Linux.bazelrc new file mode 100644 index 0000000000..d5dcf5de00 --- /dev/null +++ b/ci/Linux.bazelrc @@ -0,0 +1,3 @@ +import common.bazelrc + +build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 diff --git a/ci/Windows.bazelrc b/ci/Windows.bazelrc new file mode 100644 index 0000000000..427e923213 --- /dev/null +++ b/ci/Windows.bazelrc @@ -0,0 +1,2 @@ +import common.bazelrc + diff --git a/ci/common.bazelrc b/ci/common.bazelrc new file mode 100644 index 0000000000..c1d90ed79c --- /dev/null +++ b/ci/common.bazelrc @@ -0,0 +1,33 @@ +build:dbg --compilation_mode=dbg + +build:opt --compilation_mode=opt + +build:san-common --config=dbg --strip=never --copt=-O0 --copt=-fno-omit-frame-pointer + +build:asan --config=san-common --copt=-fsanitize=address --linkopt=-fsanitize=address +build:asan --copt=-DADDRESS_SANITIZER=1 +# ASAN hits ODR violations with shared linkage due to rules_proto. +build:asan --dynamic_mode=off + +build:msan --config=san-common --copt=-fsanitize=memory --linkopt=-fsanitize=memory +build:msan --copt=-fsanitize-memory-track-origins +build:msan --copt=-fsanitize-memory-use-after-dtor +build:msan --action_env=MSAN_OPTIONS=poison_in_dtor=1 +build:msan --copt=-DMEMORY_SANITIZER=1 + +# Use our instrumented LLVM libc++ in Kokoro. +build:kokoro-msan --config=msan +build:kokoro-msan --linkopt=-L/opt/libcxx_msan/lib +build:kokoro-msan --linkopt=-Wl,-rpath,/opt/libcxx_msan/lib +build:kokoro-msan --cxxopt=-stdlib=libc++ --linkopt=-stdlib=libc++ + + +build:tsan --config=san-common --copt=-fsanitize=thread --linkopt=-fsanitize=thread +build:tsan --copt=-DTHREAD_SANITIZER=1 + +build:ubsan --config=san-common --copt=-fsanitize=undefined --linkopt=-fsanitize=undefined +build:ubsan --action_env=UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 +build:ubsan --copt=-DUNDEFINED_SANITIZER=1 +# Workaround for the fact that Bazel links with $CC, not $CXX +# https://github.com/bazelbuild/bazel/issues/11122#issuecomment-613746748 +build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc new file mode 100644 index 0000000000..d5dcf5de00 --- /dev/null +++ b/ci/macOS.bazelrc @@ -0,0 +1,3 @@ +import common.bazelrc + +build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14 diff --git a/src/google/protobuf/io/zero_copy_stream_unittest.cc b/src/google/protobuf/io/zero_copy_stream_unittest.cc index 1cacac08d6..734543eb40 100644 --- a/src/google/protobuf/io/zero_copy_stream_unittest.cc +++ b/src/google/protobuf/io/zero_copy_stream_unittest.cc @@ -731,9 +731,11 @@ TEST_F(IoTest, StringIo) { // Verifies that outputs up to kint32max can be created. TEST_F(IoTest, LargeOutput) { - // Filter out this test on 32-bit architectures and tsan builds. + // Filter out this test on 32-bit architectures and builds where our test + // infrastructure can't handle it. if(sizeof(void*) < 8) return; -#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) +#if !defined(THREAD_SANITIZER) && !defined(MEMORY_SANITIZER) && \ + !defined(_MSC_VER) std::string str; StringOutputStream output(&str); void* unused_data;