diff --git a/.github/actions/ccache/action.yml b/.github/actions/ccache/action.yml new file mode 100644 index 0000000000..2fdcc7e5a6 --- /dev/null +++ b/.github/actions/ccache/action.yml @@ -0,0 +1,65 @@ +name: 'CCache Setup' +description: 'Run a Bazel-based docker image for Protobuf CI testing' +inputs: + cache-prefix: + required: true + description: A unique prefix to prevent cache pollution + type: string + support-modules: + required: false + description: Whether or not we need to support modules. This can result in extra cache misses. + +runs: + using: 'composite' + steps: + - name: Setup ccache on Windows + if: ${{ runner.os == 'Windows' }} + uses: ./.github/actions/internal/ccache-setup-windows + - name: Setup ccache on Mac + if: ${{ runner.os == 'macOS' }} + shell: bash + run: brew install ccache + + - name: Setup fixed path ccache caching + uses: actions/cache@627f0f41f6904a5b1efbaed9f96d9eb58e92e920 # v3.2.4 + with: + path: .ccache + # Always push to a cache key unique to this commit. + key: ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref, github.sha) }} + # Select a cache to restore from with the follow order of preference: + # 1) The exact same commit we're running over + # 2) The latest cache from the current ref branch + # 3) The latest push to the base ref of a pull request + restore-keys: | + ${{ format('ccache-{0}-{1}-{2}', inputs.cache-prefix, github.ref, github.sha) }} + ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.ref) }} + ${{ format('ccache-{0}-{1}', inputs.cache-prefix, github.base_ref) }} + + - name: Configure ccache environment variables + shell: bash + run: | + echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV + echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV + echo "CCACHE_COMPRESS=true" >> $GITHUB_ENV + echo "CCACHE_COMPRESSLEVEL=6" >> $GITHUB_ENV + echo "CCACHE_MAXSIZE=600M" >> $GITHUB_ENV + echo "CCACHE_SLOPPINESS=clang_index_store,include_file_ctime,include_file_mtime,file_macro,time_macros" >> $GITHUB_ENV + echo "CCACHE_DIRECT=true" >> $GITHUB_ENV + echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache $CCACHE_CMAKE_FLAGS" >> $GITHUB_ENV + + - name: Enable module support + if: ${{ inputs.support-modules }} + shell: bash + run: | + echo "CCACHE_SLOPPINESS=$CCACHE_SLOPPINESS,modules" >> $GITHUB_ENV + echo "CCACHE_DEPEND=true" >> $GITHUB_ENV + + - name: Zero out ccache + if: ${{ runner.os == 'macOS' }} + shell: bash + run: ccache -z + + - name: Zero out ccache + if: ${{ runner.os == 'Windows' }} + shell: pwsh + run: ${{ github.workspace }}\ccache.exe -z diff --git a/.github/actions/internal/ccache-setup-windows/action.yml b/.github/actions/internal/ccache-setup-windows/action.yml new file mode 100644 index 0000000000..5479cd3280 --- /dev/null +++ b/.github/actions/internal/ccache-setup-windows/action.yml @@ -0,0 +1,36 @@ +name: 'CCache Setup' +description: 'Setup ccache for us in Windows CI' +inputs: + ccache-version: + required: false + default: '4.7.4' + description: A pinned version of ccache + type: string + +runs: + using: 'composite' + steps: + - name: Configure ccache environment variables + shell: pwsh + run: | + Write-Host $Env:GITHUB_REF + $cllocation = (Get-Command cl.exe).Path + echo "CCACHE_COMPILER=$cllocation" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + echo "CCACHE_COMPILERTYPE=msvc" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append + + - name: Download ccache + shell: bash + run: | + curl -kLSs "https://github.com/ccache/ccache/releases/download/v${{ inputs.ccache-version }}/ccache-${{ inputs.ccache-version }}-windows-x86_64.zip" -o ccache.zip + unzip ccache.zip + cp ccache-${{ inputs.ccache-version }}-windows-x86_64/ccache.exe ccache.exe + cp ccache.exe cl.exe + rm ccache.zip + + - name: Configure msbuild flags + shell: bash + run: echo "CCACHE_MSBUILD_FLAGS=/p:CLToolExe=cl.exe /p:CLToolPath=${{ github.workspace}}" >> $GITHUB_ENV + + - name: Configure cmake flags + shell: bash + run: echo "CCACHE_CMAKE_FLAGS=-Dprotobuf_ALLOW_CCACHE=ON" >> $GITHUB_ENV diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 66c6677424..345bf0518a 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -56,39 +56,23 @@ jobs: fail-fast: false # Don't cancel all jobs if one fails. matrix: include: - - command: > + - command: >- /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 - name: Ninja - command: > + command: >- /test.sh -G Ninja -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 - name: Shared - command: > + command: >- /test.sh -Dprotobuf_BUILD_CONFORMANCE=ON -Dprotobuf_BUILD_SHARED_LIBS=ON -DCMAKE_CXX_STANDARD=14 - - name: Install - command: > - /install.sh -DCMAKE_CXX_STANDARD=14 \&\& /test.sh - -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON - -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF - -Dprotobuf_BUILD_CONFORMANCE=ON - -DCMAKE_CXX_STANDARD=14 - - name: 32-bit - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:6651a299483f7368876db7aed0802ad4ebf038d626d8995ba7df08978ff43210 - platform: linux/386 - command: >- - /bin/bash -c ' - cd /workspace; - cmake . -DCMAKE_CXX_STANDARD=14; - cmake --build . --parallel 20; - ctest --verbose --parallel 20' name: Linux CMake ${{ matrix.name}} runs-on: ubuntu-latest @@ -98,13 +82,78 @@ jobs: with: submodules: recursive ref: ${{ inputs.safe-checkout }} + + - name: Setup ccache + uses: ./.github/actions/ccache + with: + cache-prefix: linux-cmake-${{ matrix.name }} + + - name: Run tests + uses: ./.github/actions/docker + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7d5efe13c7a94f742e2f54af0550ffeec7a695e5f981ff3a0197af478f9f2a86 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: ${{ matrix.command }} ${{ env.CCACHE_CMAKE_FLAGS }} + + linux-cmake-install: + name: Linux CMake Install + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + + - name: Setup ccache + uses: ./.github/actions/ccache + with: + cache-prefix: linux-cmake-install + + - name: Run tests + uses: ./.github/actions/docker + with: + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:7d5efe13c7a94f742e2f54af0550ffeec7a695e5f981ff3a0197af478f9f2a86 + credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} + command: >- + /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} \&\& + /test.sh + ${{ env.CCACHE_CMAKE_FLAGS }} + -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON + -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF + -Dprotobuf_BUILD_CONFORMANCE=ON + -DCMAKE_CXX_STANDARD=14 + + linux-cmake-32-bit: + name: Linux CMake 32-bit + runs-on: ubuntu-latest + steps: + - name: Checkout pending changes + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + with: + submodules: recursive + ref: ${{ inputs.safe-checkout }} + + - name: Setup ccache + uses: ./.github/actions/ccache + with: + cache-prefix: linux-cmake-32-bit + - name: Run tests uses: ./.github/actions/docker with: - image: ${{ matrix.image || 'us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake@sha256:cc23dbe065668158ca2732aa305a07bd0913a175b2079d27d9c16925d23f2335' }} - platform: ${{ matrix.platform }} + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:f99f051daa8b12f4ebad5927f389bc71372f771ab080290ab451cbaf1648f9ea + platform: linux/386 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: ${{ matrix.command }} + command: >- + /bin/bash -c ' + cd /workspace; + ccache -z; + cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }}; + cmake --build . --parallel 20; + ctest --verbose --parallel 20; + ccache -s' + non-linux: strategy: fail-fast: false # Don't cancel all jobs if one fails. @@ -140,16 +189,25 @@ jobs: submodules: recursive ref: ${{ inputs.safe-checkout }} + - name: Setup ccache + uses: ./.github/actions/ccache + with: + cache-prefix: macos-cmake + - name: Configure CMake uses: ./.github/actions/bash with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: cmake . -DCMAKE_CXX_STANDARD=14 + command: cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.CCACHE_CMAKE_FLAGS }} - name: Build run: cmake --build . --parallel 8 - name: Test run: ctest --verbose -C Debug + - name: Report ccache stats + shell: bash + run: ccache -s -v + windows-cmake: strategy: fail-fast: false # Don't cancel all jobs if one fails. @@ -172,10 +230,15 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup MSVC - uses: microsoft/setup-msbuild@d3ea839497466fb4c6b91ce85831f3a251a2fe3f # v1.0.0 + uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 + with: + arch: x64 + vsversion: '2019' + + - name: Setup ccache + uses: ./.github/actions/ccache with: - vs-version: '15' - msbuild-architecture: x64 + cache-prefix: windows-cmake-${{ matrix.name }} - name: Configure CMake uses: ./.github/actions/bash @@ -183,6 +246,7 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: | cmake . -G "Visual Studio 16 2019" -A x64 \ + ${{ env.CCACHE_CMAKE_FLAGS }} \ -Dprotobuf_BUILD_CONFORMANCE=OFF \ -Dprotobuf_WITH_ZLIB=OFF \ ${{ matrix.flags }} @@ -191,10 +255,14 @@ jobs: run: >- msbuild.exe protobuf.sln /maxcpucount:8 /p:BuildInParallel=true /p:Configuration=Debug /p:Platform=x64 /p:VisualStudioVersion=15.0 + ${{ env.CCACHE_MSBUILD_FLAGS }} - name: Run Tests run: ctest --verbose -C Debug + - name: Report ccache stats + run: ${{ github.workspace }}\ccache.exe -s -v + windows-cmake-install: name: Windows CMake Install runs-on: windows-2019 @@ -206,10 +274,15 @@ jobs: ref: ${{ inputs.safe-checkout }} - name: Setup MSVC - uses: microsoft/setup-msbuild@d3ea839497466fb4c6b91ce85831f3a251a2fe3f # v1.0.0 + uses: ilammy/msvc-dev-cmd@cec98b9d092141f74527d0afa6feb2af698cfe89 # v1.12.1 + with: + arch: x64 + vsversion: '2019' + + - name: Setup ccache + uses: ./.github/actions/ccache with: - vs-version: '15' - msbuild-architecture: x64 + cache-prefix: windows-cmake - name: Configure CMake for Install uses: ./.github/actions/bash @@ -219,6 +292,7 @@ jobs: mkdir build pushd build cmake .. -G "Visual Studio 16 2019" -A x64 \ + ${{ env.CCACHE_CMAKE_FLAGS }} \ -Dprotobuf_BUILD_CONFORMANCE=OFF \ -Dprotobuf_WITH_ZLIB=OFF popd @@ -226,7 +300,7 @@ jobs: - name: Build and Install Protobuf for Windows 15 2017 run: | pushd build - msbuild.exe INSTALL.vcxproj /p:Platform=x64 /p:VisualStudioVersion=15.0 /maxcpucount:8 /p:BuildInParallel=true + msbuild.exe INSTALL.vcxproj /p:Platform=x64 /p:VisualStudioVersion=15.0 /maxcpucount:8 /p:BuildInParallel=true ${{ env.CCACHE_MSBUILD_FLAGS }} popd - name: Clear CMake cache @@ -234,19 +308,23 @@ jobs: run: rm -rf build/* - name: Configure CMake - uses: ./.github/actions/bash - with: - credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: | - cmake . -G "Visual Studio 16 2019" -A x64 \ - -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \ - -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \ - -Dprotobuf_BUILD_CONFORMANCE=OFF + shell: bash + run: | + cmake . -G "Visual Studio 16 2019" -A x64 \ + ${{ env.CCACHE_CMAKE_FLAGS }} \ + -Dprotobuf_REMOVE_INSTALLED_HEADERS=ON \ + -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF \ + -Dprotobuf_BUILD_CONFORMANCE=OFF \ + -Dprotobuf_WITH_ZLIB=OFF - name: Build for Windows 15 2017 run: >- msbuild.exe protobuf.sln /maxcpucount:8 /p:BuildInParallel=true /p:Configuration=Debug /p:Platform=x64 /p:VisualStudioVersion=15.0 + ${{ env.CCACHE_MSBUILD_FLAGS }} - name: Run Tests run: ctest --verbose -C Debug + + - name: Report ccache stats + run: ${{ github.workspace }}\ccache.exe -s -v diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml index dfcf02537b..2d93d8f2b2 100644 --- a/.github/workflows/test_php.yml +++ b/.github/workflows/test_php.yml @@ -68,7 +68,7 @@ jobs: name: Linux 32-bit ${{ matrix.version}}${{ matrix.suffix_name }}${{ matrix.test_name }} runs-on: ubuntu-latest env: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:6651a299483f7368876db7aed0802ad4ebf038d626d8995ba7df08978ff43210 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:97f50ab24582380012d7ddef5f82f08e19b9dff55d09a4a8d90a87421ae66a45 steps: - name: Checkout pending changes uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 @@ -92,7 +92,7 @@ jobs: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /bin/bash -c ' - cd /workspace/php && php -v && php -m; + cd php && php -v && php -m; composer update --ignore-platform-reqs; PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} PATH="/usr/local/php-${{ matrix.version }}${{matrix.suffix}}/bin:$PATH" diff --git a/CMakeLists.txt b/CMakeLists.txt index ca4144bee9..5c5513ca29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,7 @@ option(protobuf_BUILD_PROTOC_BINARIES "Build libprotoc and protoc compiler" ON) option(protobuf_BUILD_LIBPROTOC "Build libprotoc" OFF) option(protobuf_DISABLE_RTTI "Remove runtime type information in the binaries" OFF) option(protobuf_TEST_XML_OUTDIR "Output directory for XML logs from tests." "") +option(protobuf_ALLOW_CCACHE "Adjust build flags to allow for ccache support." OFF) if (BUILD_SHARED_LIBS) set(protobuf_BUILD_SHARED_LIBS_DEFAULT ON) else (BUILD_SHARED_LIBS) @@ -290,6 +291,16 @@ if (MSVC) string(REPLACE "/" "\\" PROTOBUF_BINARY_WIN32_PATH ${protobuf_BINARY_DIR}) string(REPLACE "." "," protobuf_RC_FILEVERSION "${protobuf_VERSION}") + if (protobuf_ALLOW_CCACHE) + # In order to support ccache, we replace the /Zi option with /Z7. This + # embeds debug symbols into the object files instead of creating a separate + # pdb file, which isn't currently supported by ccache. + string(REPLACE "/Zi" "/Z7" CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG}") + string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG}") + string(REPLACE "/Zi" "/Z7" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}") + string(REPLACE "/Zi" "/Z7" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}") + endif() + # Suppress linker warnings about files with no symbols defined. string(APPEND CMAKE_STATIC_LINKER_FLAGS " /ignore:4221") diff --git a/regenerate_stale_files.sh b/regenerate_stale_files.sh index 881ebf64a4..1c38104135 100755 --- a/regenerate_stale_files.sh +++ b/regenerate_stale_files.sh @@ -5,6 +5,8 @@ set -ex +echo "::group::Regenerate stale files" + # Cd to the repo root. cd $(dirname -- "$0") @@ -20,3 +22,5 @@ ${BazelBin} test src/google/protobuf:well_known_types_staleness_test "$@" || ./b # so just regenerating in place should be harmless. ${BazelBin} build src/google/protobuf/compiler:protoc "$@" (export PROTOC=$PWD/bazel-bin/protoc && cd csharp && ./generate_protos.sh) + +echo "::endgroup::"