Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Migrate to sccache for better CMake caching

Browse Source 
This hooks up to the changes in https://github.com/protocolbuffers/protobuf-ci/pull/26.  We see a speedup of up to 50% in no-op runs of our slowest CI builds.

PiperOrigin-RevId: 565122690
pull/14051/head
Mike Kruskal 1 year ago committed by Copybara-Service
parent 78d71cac9f
commit 4d6ad56e4a
6 changed files with 98 additions and 67 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 12
      .github/workflows/README.md
  2. 145
      .github/workflows/test_cpp.yml
  3. 2
      .github/workflows/test_java.yml
  4. 2
      .github/workflows/test_php.yml
  5. 2
      .github/workflows/test_python.yml
  6. 2
      .github/workflows/test_upb.yml

12
.github/workflows/README.md
Unescape View File

@ -153,6 +153,18 @@ While the ccache caches quickly over-run our GitHub action cache, they also
quickly become useless. Since GitHub prunes caches based on the time they were
last used, this just means that we'll see quicker turnover.
## sccache
An alternative to ccache is [sccache](https://github.com/mozilla/sccache). The
two tools are very similar in function, but sccache requires (and allows) much
less configuration and supports GCS storage right out of the box. By hooking
this up to our project that we already use for Bazel caching, we're able to get
even bigger CMake wins in CI because we're no longer constrained by GitHub's
10GB cache limit.
Similar to the Bazel remote cache, we give read access to every CI run, but
disallow writing in PRs from forks.
## Bazelisk
Bazelisk will automatically download a pinned version of Bazel on first use.

145
.github/workflows/test_cpp.yml
Unescape View File

@ -35,7 +35,7 @@ jobs:
targets: "//src/..."
- config: { name: "aarch64" }
targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e02879ddc84fa6917e4e22833e7f75884ea890b8"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17"
name: Linux ${{ matrix.config.name }}
runs-on: ubuntu-latest
steps:
@ -44,7 +44,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
uses: protocolbuffers/protobuf-ci/bazel-docker@version-test
with:
image: ${{ matrix.image }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
@ -64,9 +64,9 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
uses: protocolbuffers/protobuf-ci/bazel-docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-518b4fcd8d0ded2484c94f02e835526cacfdac2d
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: cpp_linux/gcc-${{ matrix.version }}
bazel: test //pkg/... //src/... @com_google_protobuf_examples//...
@ -86,28 +86,29 @@ jobs:
submodules: recursive
- name: Cross compile protoc for ${{ matrix.arch }}
id: cross-compile
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v2
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:6.3.0-91a0ac83e968068672bc6001a4d474cfd9a50f1d
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
architecture: linux-${{ matrix.arch }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-release-${{ matrix.arch }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:${{ matrix.arch }}-e02879ddc84fa6917e4e22833e7f75884ea890b8
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:${{ matrix.arch }}-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
entrypoint: bash
command: >
-c "set -ex;
ccache -z;
cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_WITH_ZLIB=OFF ${{ env.CCACHE_CMAKE_FLAGS }};
sccache -z;
cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake --build . --parallel 20;
ctest --parallel 20;
ccache -s -v"
sccache -s"
linux-cmake:
strategy:
@ -133,18 +134,19 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-cmake-${{ matrix.name }}
cache-prefix: linux-cmake
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-e508da4f6a96f1baeca8aafa4d835c48295691b3
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/test.sh ${{ matrix.flags}} ${{ env.CCACHE_CMAKE_FLAGS }}
/test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }}
-Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON
-Dprotobuf_ABSL_PROVIDER=package
@ -158,20 +160,21 @@ jobs:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-cmake-install
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-e508da4f6a96f1baeca8aafa4d835c48295691b3
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package \&\&
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package \&\&
/test.sh
${{ env.CCACHE_CMAKE_FLAGS }}
${{ env.SCCACHE_CMAKE_FLAGS }}
-Dprotobuf_REMOVE_INSTALLED_HEADERS=ON
-Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
-Dprotobuf_BUILD_CONFORMANCE=ON
@ -187,18 +190,19 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-cmake-examples
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-e508da4f6a96f1baeca8aafa4d835c48295691b3
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }}
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
-Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
-Dprotobuf_BUILD_EXAMPLES=OFF \&\&
mkdir examples/build \&\&
@ -227,25 +231,26 @@ jobs:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-cmake-gcc-${{ matrix.name }}
cache-prefix: linux-cmake-gcc
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-518b4fcd8d0ded2484c94f02e835526cacfdac2d
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
entrypoint: bash
command: >-
-c 'set -ex;
cd /workspace;
ccache -z;
cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }};
sccache -z;
cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake --build . --parallel 20;
ctest --verbose --parallel 20;
ccache -s'
sccache -s'
linux-cmake-submodules:
name: Linux CMake Submodules
@ -257,18 +262,19 @@ jobs:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-cmake-${{ matrix.name }}
cache-prefix: linux-cmake-submodules
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-e508da4f6a96f1baeca8aafa4d835c48295691b3
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-63dd26c0c7a808d92673a3e52e848189d4ab0f17
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/test.sh ${{ env.CCACHE_CMAKE_FLAGS }}
/test.sh ${{ env.SCCACHE_CMAKE_FLAGS }}
-Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
linux-cmake-32-bit:
@ -281,25 +287,26 @@ jobs:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: linux-cmake-32-bit
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v2
uses: protocolbuffers/protobuf-ci/docker@version-test
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:8275360dc5d676f3470872d79087901c0e4153453976bea908a92c82e8d209ea
platform: linux/386
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/bin/bash -cex '
cd /workspace;
ccache -z;
cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }};
sccache -z;
cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake --build . --parallel 20;
ctest --verbose --parallel 20;
ccache -s'
sccache -s'
non-linux:
strategy:
@ -328,7 +335,7 @@ jobs:
with:
ref: ${{ inputs.safe-checkout }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel@v2
uses: protocolbuffers/protobuf-ci/bazel@version-test
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel: ${{ matrix.bazel }}
@ -342,6 +349,7 @@ jobs:
- name: MacOS CMake
os: macos-12
flags: -DCMAKE_CXX_STANDARD=14
cache-prefix: macos-cmake
- name: Windows CMake
os: windows-2019
flags: >-
@ -349,6 +357,7 @@ jobs:
-Dprotobuf_BUILD_SHARED_LIBS=OFF
-Dprotobuf_BUILD_EXAMPLES=ON
vsversion: '2019'
cache-prefix: windows-2019-cmake
- name: Windows CMake 2022
os: windows-2022
flags: >-
@ -356,18 +365,21 @@ jobs:
-Dprotobuf_BUILD_SHARED_LIBS=OFF
-Dprotobuf_BUILD_EXAMPLES=ON
vsversion: '2022'
cache-prefix: windows-2022-cmake
- name: Windows CMake 32-bit
os: windows-2019
flags: >-
-G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
vsversion: '2019'
windows-arch: 'win32'
cache-prefix: windows-2019-win32-cmake
- name: Windows CMake Shared
os: windows-2019
flags: >-
-G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF
-Dprotobuf_BUILD_SHARED_LIBS=ON
vsversion: '2019'
cache-prefix: windows-2019-cmake
- name: Windows CMake Install
os: windows-2019
install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF
@ -376,6 +388,7 @@ jobs:
-Dprotobuf_REMOVE_INSTALLED_HEADERS=ON
-Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
vsversion: '2019'
cache-prefix: windows-2019-cmake
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
steps:
@ -385,20 +398,26 @@ jobs:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup ccache
uses: protocolbuffers/protobuf-ci/ccache@v2
- name: Setup MSVC
if: ${{ runner.os == 'Windows' }}
uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1
with:
cache-prefix: ${{ matrix.name }}
arch: ${{ matrix.windows-arch || 'x64' }}
vsversion: ${{ matrix.vsversion }}
windows-arch: ${{ matrix.windows-arch || 'win64' }}
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@version-test
with:
cache-prefix: ${{ matrix.cache-prefix }}
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
# Install phase.
- name: Configure CMake for install
if: matrix.install-flags
uses: protocolbuffers/protobuf-ci/bash@v2
uses: protocolbuffers/protobuf-ci/bash@version-test
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }}
command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
- name: Build for install
if: matrix.install-flags
shell: bash
@ -407,20 +426,20 @@ jobs:
if: matrix.install-flags
shell: bash
run: cmake --build . --target install
- name: Report and clear ccache stats
- name: Report and clear sccache stats
if: matrix.install-flags
shell: bash
run: ccache -s -v && ccache -z
run: sccache -s && sccache -z
- name: Clear CMake cache
if: matrix.install-flags
shell: bash
run: cmake --build . --target clean && rm CMakeCache.txt
- name: Configure CMake
uses: protocolbuffers/protobuf-ci/bash@v2
uses: protocolbuffers/protobuf-ci/bash@version-test
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }}
command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
- name: Build
shell: bash
@ -430,6 +449,6 @@ jobs:
shell: bash
run: ctest --verbose --parallel 20 -C Debug
- name: Report ccache stats
- name: Report sccache stats
shell: bash
run: ccache -s -v
run: sccache -s

2
.github/workflows/test_java.yml
Unescape View File

@ -31,7 +31,7 @@ jobs:
targets: //java/... //java/internal:java_version
- name: aarch64
version: 'aarch64'
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e02879ddc84fa6917e4e22833e7f75884ea890b8
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17
targets: //java/... //src/google/protobuf/compiler:protoc_aarch64_test
name: Linux ${{ matrix.name }}

2
.github/workflows/test_php.yml
Unescape View File

@ -75,7 +75,7 @@ jobs:
name: Linux 32-bit ${{ matrix.version}}${{ matrix.suffix_name }}${{ matrix.test_name }}
runs-on: ubuntu-latest
env:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:97f50ab24582380012d7ddef5f82f08e19b9dff55d09a4a8d90a87421ae66a45
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:8275360dc5d676f3470872d79087901c0e4153453976bea908a92c82e8d209ea
steps:
- name: Checkout pending changes
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0

2
.github/workflows/test_python.yml
Unescape View File

@ -30,7 +30,7 @@ jobs:
targets: //python/... //python:aarch64_test
# TODO(b/262628111) Enable this once conformance tests are fixed.
flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-e02879ddc84fa6917e4e22833e7f75884ea890b8
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17
name: Linux ${{ matrix.type }} ${{ matrix.version }}
runs-on: ubuntu-latest

2
.github/workflows/test_upb.yml
Unescape View File

@ -55,7 +55,7 @@ jobs:
- name: Run tests
uses: protocolbuffers/protobuf-ci/bazel-docker@v2
with:
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-518b4fcd8d0ded2484c94f02e835526cacfdac2d"
image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:12.2-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17"
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
bazel-cache: "upb-bazel-gcc"
bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt //upb/...

Write Preview
Loading…
Cancel
Save