From 9c2211be08af10dd5964cf8b65364afaef001fdf Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Fri, 10 Feb 2023 18:24:16 -0800 Subject: [PATCH] Switch to Ninja generator for windows cmake builds. These will still use MSVC as the compiler, but will no longer generate Visual Studio projects for the builds. Visual Studio is particularly bad at parallelizing builds, and is hostile to ccache. This change also tweaks the ccache setup to prevent unbounded the growth observed in our github caches. Windows builds have had debug symbols stripped to reduce ccache size by a factor of 2x, and ccache maximum have been tweaked so that we persist fewer older builds. Before this change, each CMake build took 12 minutes on every run (plus some constant overhead from staleness/gcloud), even with caching or on large multi-core runners. No amount of caching or parallelization made any noticeable difference above noise. With this change, we see the following improvements: - 12 minutes to build from scratch on normal runners (unchanged) - 4 minutes on 32-core runners from scratch - 1 minute with optimal caches available on normal runners. - 30 seconds on 32-core runners with optimal caches PiperOrigin-RevId: 508799909 --- .github/actions/ccache/action.yml | 36 ++-- .../internal/ccache-setup-windows/action.yml | 28 +-- .github/workflows/test_cpp.yml | 166 ++++++------------ CMakeLists.txt | 16 +- 4 files changed, 90 insertions(+), 156 deletions(-) diff --git a/.github/actions/ccache/action.yml b/.github/actions/ccache/action.yml index 2a100ccb7f..2d70550364 100644 --- a/.github/actions/ccache/action.yml +++ b/.github/actions/ccache/action.yml @@ -12,6 +12,18 @@ inputs: runs: using: 'composite' steps: + - name: Configure ccache environment variables + shell: bash + run: | + echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=5" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=100M" >> $GITHUB_ENV + echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV + echo "CCACHE_DIRECT=true" >> $GITHUB_ENV + echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" >> $GITHUB_ENV + - name: Setup ccache on Windows if: ${{ runner.os == 'Windows' }} uses: ./.github/actions/internal/ccache-setup-windows @@ -23,7 +35,10 @@ runs: - name: Setup fixed path ccache caching uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 with: - path: .ccache + path: | + .ccache/** + !.ccache/lock + !.ccache/tmp # Always push to a cache key unique to this commit. key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref_name, github.sha) }} # Select a cache to restore from with the follow order of preference: @@ -35,18 +50,6 @@ runs: ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref_name) }} ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }} - - name: Configure ccache environment variables - shell: bash - run: | - echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV - echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV - echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV - echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV - echo "CCACHE_MAXSIZE=600M" >> $GITHUB_ENV - echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV - echo "CCACHE_DIRECT=true" >> $GITHUB_ENV - echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache $CCACHE_CMAKE_FLAGS" >> $GITHUB_ENV - - name: Enable module support if: ${{ inputs.support-modules }} shell: bash @@ -55,11 +58,6 @@ runs: echo "CCACHE_DEPEND=true" >> $GITHUB_ENV - name: Zero out ccache - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os != 'Linux' }} shell: bash run: ccache -z - - - name: Zero out ccache - if: ${{ runner.os == 'Windows' }} - shell: pwsh - run: ${{ github.workspace }}\ccache.exe -z diff --git a/.github/actions/internal/ccache-setup-windows/action.yml b/.github/actions/internal/ccache-setup-windows/action.yml index 5479cd3280..a42e7bf8a8 100644 --- a/.github/actions/internal/ccache-setup-windows/action.yml +++ b/.github/actions/internal/ccache-setup-windows/action.yml @@ -10,6 +10,16 @@ inputs: runs: using: 'composite' steps: + - name: Setup MSVC + uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 + with: + arch: x64 + vsversion: '2019' + + - name: Install ccache + shell: bash + run: choco install ccache --version=4.7.4 + - name: Configure ccache environment variables shell: pwsh run: | @@ -18,19 +28,9 @@ runs: echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append - - name: Download ccache + - name: Configure Windows-specific ccache environment variables shell: bash + # Windows caches are about 2x larger than other platforms. run: | - curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip - unzip ccache.zip - cp ccache-${{ inputs.ccache-version }}-windows-x86_64/ccache.exe ccache.exe - cp ccache.exe cl.exe - rm ccache.zip - - - name: Configure msbuild flags - shell: bash - run: echo "CCACHE_MSBUILD_FLAGS=/p:CLToolExe=cl.exe /p:CLToolPath=${{ github.workspace}}" >> $GITHUB_ENV - - - name: Configure cmake flags - shell: bash - run: echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=10" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=200M" >> $GITHUB_ENV diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 5d51bcae3c..f846d80a8d 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -179,49 +179,34 @@ jobs: bazel: test ${{ matrix.bazel }} bazel-cache: cpp_${{ matrix.os }} - macos-cmake: - name: MacOS CMake - runs-on: macos-12 - steps: - - name: Checkout pending changes - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - with: - submodules: recursive - ref: ${{ inputs.safe-checkout }} - - - name: Setup ccache - uses: ./.github/actions/ccache - with: - cache-prefix: macos-cmake - - - name: Configure CMake - uses: ./.github/actions/bash - with: - credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} - - name: Build - run: cmake --build . --parallel 8 - - name: Test - run: ctest --verbose --parallel 20 -C Debug - - - name: Report ccache stats - shell: bash - run: ccache -s -v - - windows-cmake: + non-linux-cmake: strategy: fail-fast: false # Don't cancel all jobs if one fails. matrix: include: - - name: Visual Studio + - name: MacOS CMake + os: macos-12 + flags: -DCMAKE_CXX_STANDARD=14 + - name: Windows CMake + os: windows-2019 flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF -Dprotobuf_BUILD_EXAMPLES=ON - - name: Shared - flags: -Dprotobuf_BUILD_SHARED_LIBS=ON - - name: Windows CMake ${{ matrix.name}} - runs-on: windows-2019 + - name: Windows CMake Shared + os: windows-2019 + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_BUILD_SHARED_LIBS=ON + - name: Windows CMake Install + os: windows-2019 + install-flags: -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF -Dprotobuf_BUILD_TESTS=OFF + flags: >- + -G Ninja -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_BUILD_CONFORMANCE=OFF + -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON + -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} steps: - name: Checkout pending changes uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 @@ -229,102 +214,49 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 - with: - arch: x64 - vsversion: '2019' - - name: Setup ccache uses: ./.github/actions/ccache with: - cache-prefix: windows-cmake-${{ matrix.name }} + cache-prefix: ${{ matrix.name }} - - name: Configure CMake + # Install phase. + - name: Configure CMake for install + if: matrix.install-flags uses: ./.github/actions/bash with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: | - cmake . -G "Visual Studio 16 2019" -A x64 \ - ${{ env.CCACHE_CMAKE_FLAGS }} \ - -Dprotobuf_BUILD_CONFORMANCE=OFF \ - -Dprotobuf_WITH_ZLIB=OFF \ - ${{ matrix.flags }} - - - name: Build for Windows 15 2017 - run: >- - msbuild.exe protobuf.sln /p:MultiProcessorCompilation=true /p:CL_MPCount=8 /maxcpucount:8 /p:BuildInParallel=true - /p:Configuration=Debug /p:Platform=x64 /p:VisualStudioVersion=15.0 - ${{ env.CCACHE_MSBUILD_FLAGS }} - - - name: Run Tests - run: ctest --verbose --parallel 20 -C Debug - - - name: Report ccache stats - run: ${{ github.workspace }}\ccache.exe -s -v - - windows-cmake-install: - name: Windows CMake Install - runs-on: windows-2019 - steps: - - name: Checkout pending changes - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - with: - submodules: recursive - ref: ${{ inputs.safe-checkout }} - - - name: Setup MSVC - uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 - with: - arch: x64 - vsversion: '2019' - - - name: Setup ccache - uses: ./.github/actions/ccache - with: - cache-prefix: windows-cmake + command: cmake . ${{ matrix.install-flags }} ${{ env.CCACHE_CMAKE_FLAGS }} + - name: Build for install + if: matrix.install-flags + shell: bash + run: VERBOSE=1 cmake --build . --parallel 20 + - name: Install + if: matrix.install-flags + shell: bash + run: cmake --build . --target install + - name: Report and clear ccache stats + if: matrix.install-flags + shell: bash + run: ccache -s -v && ccache -z + - name: Clear CMake cache + if: matrix.install-flags + shell: bash + run: cmake --build . --target clean && rm CMakeCache.txt - - name: Configure CMake for Install + - name: Configure CMake uses: ./.github/actions/bash with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: | - mkdir build - pushd build - cmake .. -G "Visual Studio 16 2019" -A x64 \ - ${{ env.CCACHE_CMAKE_FLAGS }} \ - -Dprotobuf_BUILD_CONFORMANCE=OFF \ - -Dprotobuf_WITH_ZLIB=OFF - popd - - - name: Build and Install Protobuf for Windows 15 2017 - run: | - pushd build - msbuild.exe INSTALL.vcxproj /p:Platform=x64 /p:VisualStudioVersion=15.0 /p:MultiProcessorCompilation=true /p:CL_MPCount=8 /maxcpucount:8 /p:BuildInParallel=true ${{ env.CCACHE_MSBUILD_FLAGS }} - popd + command: cmake . ${{ matrix.flags }} ${{ env.CCACHE_CMAKE_FLAGS }} - - name: Clear CMake cache + - name: Build shell: bash - run: rm -rf build/* + run: VERBOSE=1 cmake --build . --parallel 20 - - name: Configure CMake + - name: Test shell: bash - run: | - cmake . -G "Visual Studio 16 2019" -A x64 \ - ${{ env.CCACHE_CMAKE_FLAGS }} \ - -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \ - -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \ - -Dprotobuf_BUILD_CONFORMANCE=OFF \ - -Dprotobuf_WITH_ZLIB=OFF - - - name: Build for Windows 15 2017 - run: >- - msbuild.exe protobuf.sln /p:MultiProcessorCompilation=true /p:CL_MPCount=8 /maxcpucount:8 /p:BuildInParallel=true - /p:Configuration=Debug /p:Platform=x64 /p:VisualStudioVersion=15.0 - ${{ env.CCACHE_MSBUILD_FLAGS }} - - - name: Run Tests run: ctest --verbose --parallel 20 -C Debug - name: Report ccache stats - run: ${{ github.workspace }}\ccache.exe -s -v + shell: bash + run: ccache -s -v diff --git a/CMakeLists.txt b/CMakeLists.txt index 5c5513ca29..20786bd3ff 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,13 +292,17 @@ if (MSVC) string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") if (protobuf_ALLOW_CCACHE) - # In order to support ccache, we replace the /Zi option with /Z7. This + # In order to support ccache, we need to remove the /Zi option because it + # puts debug symbols into separate pdb files (which in incompatible with + # ccache). This can be replaced with /Z7 to preserve debug symbols, which # embeds debug symbols into the object files instead of creating a separate - # pdb file, which isn't currently supported by ccache. - string(REPLACE "/Zi" "/Z7" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") - string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") - string(REPLACE "/Zi" "/Z7" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") - string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") + # pdb file, which isn't currently supported by ccache. However, this bloats + # the ccache size by about a factor of 2x, making it very expensive in CI. + # Instead, we strip debug symbols to reduce this overhead. + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") + string(REPLACE "/Zi" "/DEBUG:NONE" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") endif() # Suppress linker warnings about files with no symbols defined.