From d3735bc2a0ae3f0b69f668e6a0c9108d0e92fdfa Mon Sep 17 00:00:00 2001 From: Mike Kruskal Date: Tue, 15 Oct 2024 15:47:15 -0700 Subject: [PATCH] Support fetching CMake dependencies via `-Dprotobuf_FETCH_DEPENDENCIES=ON` This ports upb's WORKSPACE scraping logic to protobuf, and allows us to dynamically fetch our dependencies at the exact same pinned version as in Bazel via protobuf_FETCH_DEPENDENCIES=ON. This is mostly for development purposes, and is preferable to git submodules. In a later cl we will flip the default behavior to "package" #test-continuous PiperOrigin-RevId: 686265348 --- .github/workflows/test_cpp.yml | 65 +++--- BUILD.bazel | 55 +---- CMakeLists.txt | 18 +- MODULE.bazel | 5 +- cmake/BUILD.bazel | 25 +++ cmake/abseil-cpp.cmake | 18 +- cmake/conformance.cmake | 30 ++- cmake/dependencies_generator.py | 143 +++++++++++++ cmake/gtest.cmake | 11 + regenerate_stale_files.sh | 1 + upb/cmake/BUILD.bazel | 56 ------ upb/cmake/README.md | 23 --- upb/cmake/make_cmakelists.py | 346 -------------------------------- upb/cmake/push_auto_update.sh | 73 ------- 14 files changed, 266 insertions(+), 603 deletions(-) create mode 100644 cmake/BUILD.bazel create mode 100644 cmake/dependencies_generator.py delete mode 100644 upb/cmake/README.md delete mode 100755 upb/cmake/make_cmakelists.py delete mode 100755 upb/cmake/push_auto_update.sh diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 0b7a401e3f..2e8d11af83 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -107,7 +107,6 @@ jobs: uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - submodules: recursive - name: Cross compile protoc for ${{ matrix.arch }} id: cross-compile uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3 @@ -131,7 +130,7 @@ jobs: sccache -z; cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} -Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 - -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }}; + -Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }}; cmake --build . --parallel 20; ctest --parallel 20; sccache -s" @@ -141,18 +140,19 @@ jobs: fail-fast: false # Don't cancel all jobs if one fails. matrix: include: - - flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 + - flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package - name: Ninja - flags: -G Ninja -DCMAKE_CXX_STANDARD=14 + flags: -G Ninja -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package continuous-only: true - name: Shared - flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 + flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package continuous-only: true - name: C++17 - flags: -DCMAKE_CXX_STANDARD=17 - # TODO Re-enable this. - #- name: C++20 - # flags: -DCMAKE_CXX_STANDARD=20 + flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package + - name: C++20 + flags: -DCMAKE_CXX_STANDARD=20 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package + - name: Fetch + flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_FETCH_DEPENDENCIES=ON name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux CMake ${{ matrix.name}} runs-on: ubuntu-latest @@ -174,37 +174,51 @@ jobs: if: ${{ !matrix.continuous-only || inputs.continuous-run }} uses: protocolbuffers/protobuf-ci/docker@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }} - -Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON - -Dprotobuf_ABSL_PROVIDER=package + -Dprotobuf_BUILD_TESTS=ON ${{ matrix.package_flags }} linux-cmake-install: - name: Linux CMake Install + strategy: + fail-fast: false # Don't cancel all jobs if one fails. + matrix: + type: [package, fetch] + include: + # Set defaults + - type: package + name: Install + flags: -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package + - type: fetch + name: Install (Fetch) + flags: -Dprotobuf_FETCH_DEPENDENCIES=ON + continuous-only: true + name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }}Linux CMake ${{ matrix.name }}) runs-on: ubuntu-latest steps: - name: Checkout pending changes uses: protocolbuffers/protobuf-ci/checkout@v3 + if: ${{ !matrix.continuous-only || inputs.continuous-run }} with: ref: ${{ inputs.safe-checkout }} - submodules: recursive - name: Setup sccache uses: protocolbuffers/protobuf-ci/sccache@v3 + if: ${{ !matrix.continuous-only || inputs.continuous-run }} with: cache-prefix: linux-cmake-install credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v3 + if: ${{ !matrix.continuous-only || inputs.continuous-run }} with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }} - -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package + ${{ matrix.flags }} -Dprotobuf_BUILD_SHARED_LIBS=ON \&\& /test.sh ${{ env.SCCACHE_CMAKE_FLAGS }} @@ -212,7 +226,7 @@ jobs: -Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14 - -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package + ${{ matrix.flags }} # This test should always be skipped on presubmit linux-cmake-examples: @@ -236,7 +250,7 @@ jobs: if: ${{ inputs.continuous-run }} uses: protocolbuffers/protobuf-ci/docker@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }} @@ -268,7 +282,6 @@ jobs: uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - submodules: recursive - name: Setup sccache if: ${{ !matrix.continuous-only || inputs.continuous-run }} @@ -288,7 +301,7 @@ jobs: -c 'set -ex; cd /workspace; sccache -z; - cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }}; + cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_FETCH_DEPENDENCIES=ON; cmake --build . --parallel 20; ctest --verbose --parallel 20; sccache -s' @@ -312,7 +325,7 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /test.sh ${{ env.SCCACHE_CMAKE_FLAGS }} @@ -326,7 +339,6 @@ jobs: uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - submodules: recursive - name: Setup sccache uses: protocolbuffers/protobuf-ci/sccache@v3 @@ -337,14 +349,14 @@ jobs: - name: Run tests uses: protocolbuffers/protobuf-ci/docker@v3 with: - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:429f924aec315704b4233adcbe4b29006116f27769db98acd176b9eb69c31299 + image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:56548bef786201330017eae685cc3d2fdb564fd2ca3b88e30e28d84572e4c5dd platform: linux/386 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} command: >- /bin/bash -cex ' cd /workspace; sccache -z; - cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}; + cmake . -DCMAKE_CXX_STANDARD=14 -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }}; cmake --build . --parallel 20; ctest --verbose --parallel 20; sccache -s' @@ -460,7 +472,6 @@ jobs: uses: protocolbuffers/protobuf-ci/checkout@v3 with: ref: ${{ inputs.safe-checkout }} - submodules: recursive - name: Setup MSVC if: ${{ runner.os == 'Windows' && (!matrix.continuous-only || inputs.continuous-run) }} @@ -499,7 +510,7 @@ jobs: uses: protocolbuffers/protobuf-ci/bash@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON + command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON - name: Build for install if: ${{ matrix.install-flags && (!matrix.continuous-only || inputs.continuous-run) }} shell: bash @@ -522,7 +533,7 @@ jobs: uses: protocolbuffers/protobuf-ci/bash@v3 with: credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} - command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON + command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON - name: Build if: ${{ !matrix.continuous-only || inputs.continuous-run }} diff --git a/BUILD.bazel b/BUILD.bazel index 32b26cbdc9..97d1ef84b9 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -24,6 +24,11 @@ license( license_text = ":LICENSE", ) +exports_files( + ["MODULE.bazel"], + visibility = ["//cmake:__pkg__"], +) + ################################################################################ # Well Known Types Proto Library Rules # @@ -642,53 +647,3 @@ filegroup( srcs = glob(["**/*.bzl"]), visibility = ["//visibility:public"], ) - -################################################################################ -# Packaging rules -################################################################################ - -# Files included in all source distributions -pkg_files( - name = "common_dist_files", - srcs = glob( - [ - "*.bzl", - "cmake/*.cmake", - "cmake/*.in", - "editors/*", - ], - allow_empty = True, - ) + [ - "BUILD.bazel", - "CMakeLists.txt", - "CONTRIBUTORS.txt", - "LICENSE", - "README.md", - "WORKSPACE", - "cmake/README.md", - "generate_descriptor_proto.sh", - "maven_install.json", - "//third_party:BUILD.bazel", - "//third_party:zlib.BUILD", - ], - strip_prefix = strip_prefix.from_root(""), - visibility = ["//pkg:__pkg__"], -) - -# Additional files for C# -pkg_files( - name = "csharp_dist_files", - srcs = [ - "global.json", - ], - visibility = ["//pkg:__pkg__"], -) - -# Additional files for ObjC -pkg_files( - name = "objectivec_dist_files", - srcs = [ - "Protobuf.podspec", - ], - visibility = ["//pkg:__pkg__"], -) diff --git a/CMakeLists.txt b/CMakeLists.txt index 087b7e37cd..1cebc038d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,14 +153,6 @@ file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/cmaketest.map) find_package(Threads REQUIRED) -# We can install dependencies from submodules if we're running -# CMake v3.13 or newer. -if(CMAKE_VERSION VERSION_LESS 3.13) - set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE OFF) -else() - set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE ON) -endif() - set(_protobuf_FIND_ZLIB) if (protobuf_WITH_ZLIB) find_package(ZLIB) @@ -289,11 +281,13 @@ include_directories( ${protobuf_BINARY_DIR}/src ${protobuf_SOURCE_DIR}/src) -set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library") -set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package") +set(protobuf_FETCH_DEPENDENCIES OFF CACHE BOOL "Download dependencies from GitHub. If this option is not set, the dependency must be available locally, either as a sub-module or an installed package.") + +set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library. `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub") +set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package" "fetch") -set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library") -set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package") +set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library. `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub") +set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package" "fetch") if (protobuf_BUILD_TESTS) include(${protobuf_SOURCE_DIR}/cmake/gtest.cmake) diff --git a/MODULE.bazel b/MODULE.bazel index 5c1a018147..bfb44ba3b5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -12,7 +12,7 @@ module( # Bzlmod follows MVS: # https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution # Thus the highest version in their module graph is resolved. -bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl") +bazel_dep(name = "abseil-cpp", version = "20230802.1", repo_name = "com_google_absl") bazel_dep(name = "bazel_skylib", version = "1.7.0") bazel_dep(name = "jsoncpp", version = "1.9.5") bazel_dep(name = "rules_cc", version = "0.0.13") @@ -27,10 +27,9 @@ bazel_dep(name = "rules_rust", version = "0.51.0") bazel_dep(name = "platforms", version = "0.0.8") bazel_dep(name = "zlib", version = "1.3.1") bazel_dep(name = "bazel_features", version = "1.17.0", repo_name = "proto_bazel_features") - bazel_dep( name = "rules_shell", - version = "0.2.0" + version = "0.2.0", ) # Proto toolchains diff --git a/cmake/BUILD.bazel b/cmake/BUILD.bazel new file mode 100644 index 0000000000..375d7ac433 --- /dev/null +++ b/cmake/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_python//python:defs.bzl", "py_binary") +load("//upb/cmake:build_defs.bzl", "staleness_test") + +py_binary( + name = "dependencies_generator", + srcs = ["dependencies_generator.py"], +) + +genrule( + name = "generate_dependencies", + srcs = ["//:MODULE.bazel"], + outs = ["generated-in/dependencies.cmake"], + cmd = "$(location :dependencies_generator) " + + "$(location //:MODULE.bazel) $@", + tools = [":dependencies_generator"], +) + +staleness_test( + name = "test_dependencies_staleness", + outs = [ + "dependencies.cmake", + ], + generated_pattern = "generated-in/%s", + tags = ["manual"], +) diff --git a/cmake/abseil-cpp.cmake b/cmake/abseil-cpp.cmake index 1b64affa50..688ca49fa2 100644 --- a/cmake/abseil-cpp.cmake +++ b/cmake/abseil-cpp.cmake @@ -13,6 +13,20 @@ endif() if(TARGET absl::strings) # If Abseil is included already, skip including it. # (https://github.com/protocolbuffers/protobuf/issues/10435) +elseif (protobuf_FETCH_DEPENDENCIES OR protobuf_ABSL_PROVIDER STREQUAL "fetch") + include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake) + include(FetchContent) + FetchContent_Declare( + absl + GIT_REPOSITORY "https://github.com/abseil/abseil-cpp.git" + GIT_TAG "${abseil-cpp-version}" + ) + if(protobuf_INSTALL) + # When protobuf_INSTALL is enabled and Abseil will be built as a module, + # Abseil will be installed along with protobuf for convenience. + set(ABSL_ENABLE_INSTALL ON) + endif() + FetchContent_MakeAvailable(absl) elseif(protobuf_ABSL_PROVIDER STREQUAL "module") if(NOT ABSL_ROOT_DIR) set(ABSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp) @@ -27,10 +41,6 @@ elseif(protobuf_ABSL_PROVIDER STREQUAL "module") else() message(WARNING "protobuf_ABSL_PROVIDER is \"module\" but ABSL_ROOT_DIR is wrong") endif() - if(protobuf_INSTALL AND NOT _protobuf_INSTALL_SUPPORTED_FROM_MODULE) - message(WARNING "protobuf_INSTALL will be forced to FALSE because protobuf_ABSL_PROVIDER is \"module\" and CMake version (${CMAKE_VERSION}) is less than 3.13.") - set(protobuf_INSTALL FALSE) - endif() elseif(protobuf_ABSL_PROVIDER STREQUAL "package") # Use "CONFIG" as there is no built-in cmake module for absl. find_package(absl REQUIRED CONFIG) diff --git a/cmake/conformance.cmake b/cmake/conformance.cmake index 37fe71559a..87961df7fa 100644 --- a/cmake/conformance.cmake +++ b/cmake/conformance.cmake @@ -1,4 +1,17 @@ -if (protobuf_JSONCPP_PROVIDER STREQUAL "module") +# Don't run jsoncpp tests. +set(JSONCPP_WITH_TESTS OFF) + +if (protobuf_FETCH_DEPENDENCIES OR protobuf_JSONCPP_PROVIDER STREQUAL "fetch") + include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake) + include(FetchContent) + FetchContent_Declare( + jsoncpp + GIT_REPOSITORY "https://github.com/open-source-parsers/jsoncpp.git" + # TODO Use ${jsoncpp-version} here once it supports cmake. + GIT_TAG "1.9.4" + ) + FetchContent_MakeAvailable(jsoncpp) +elseif (protobuf_JSONCPP_PROVIDER STREQUAL "module") if (NOT EXISTS "${protobuf_SOURCE_DIR}/third_party/jsoncpp/CMakeLists.txt") message(FATAL_ERROR "Cannot find third_party/jsoncpp directory that's needed to " @@ -129,18 +142,17 @@ add_test(NAME conformance_cpp_test --text_format_failure_list ${protobuf_SOURCE_DIR}/conformance/text_format_failure_list_cpp.txt --output_dir ${protobuf_TEST_XML_OUTDIR} --maximum_edition 2023 - ${CMAKE_CURRENT_BINARY_DIR}/conformance_cpp + $ DEPENDS conformance_test_runner conformance_cpp) set(JSONCPP_WITH_TESTS OFF CACHE BOOL "Disable tests") -if(protobuf_JSONCPP_PROVIDER STREQUAL "module") +if(NOT protobuf_FETCH_DEPENDENCIES AND protobuf_JSONCPP_PROVIDER STREQUAL "module") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/jsoncpp third_party/jsoncpp) target_include_directories(conformance_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/jsoncpp/include) - if(BUILD_SHARED_LIBS) - target_link_libraries(conformance_test_runner jsoncpp_lib) - else() - target_link_libraries(conformance_test_runner jsoncpp_static) - endif() +endif() + +if(BUILD_SHARED_LIBS) + target_link_libraries(conformance_test_runner jsoncpp_lib) else() - target_link_libraries(conformance_test_runner jsoncpp) + target_link_libraries(conformance_test_runner jsoncpp_static) endif() diff --git a/cmake/dependencies_generator.py b/cmake/dependencies_generator.py new file mode 100644 index 0000000000..b7b2deeb1f --- /dev/null +++ b/cmake/dependencies_generator.py @@ -0,0 +1,143 @@ +#!/usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2023 Google LLC. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google LLC nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""A tool to convert MODULE.bazel -> CMakeLists.txt. + +This tool is very protobuf-specific at the moment, and should not be seen as a +generic Bazel -> CMake converter. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import sys +import textwrap + + +class ExtensionFunctions(object): + """A fake extension that we can use to get the functions we need.""" + + def toolchain(self, *args, **kwargs): + pass + + def parse(self, *args, **kwargs): + pass + + def spec(self, *args, **kwargs): + pass + + def from_specs(self, *args, **kwargs): + pass + + def install(self, *args, **kwargs): + pass + + +class ModuleFileFunctions(object): + """A fake MODULE file that we can exec() to get the functions we need.""" + + def __init__(self, converter): + self.converter = converter + + def module(self, *args, **kwargs): + pass + + def bazel_dep(self, name, version, **kwargs): + self.converter.toplevel += textwrap.dedent( + """\ + set(%(name)s-version "%(version)s") + """ + % { + "name": name, + "version": version, + } + ) + + def register_toolchains(self, *args): + pass + + def use_repo(self, *args, **kwargs): + pass + + def use_extension(self, *args): + return ExtensionFunctions() + + +class Converter(object): + + def __init__(self): + self.toplevel = "" + self.if_lua = "" + + def convert(self): + return self.template % { + "toplevel": converter.toplevel, + } + + template = textwrap.dedent("""\ + # Auto-generated by @//cmake:make_dependencies + # + # This file contains lists of external dependencies based on our Bazel + # config. It should be included from a hand-written CMake file that uses + # them. + # + # Changes to this file will be overwritten based on Bazel definitions. + + if(${CMAKE_VERSION} VERSION_GREATER 3.10 OR ${CMAKE_VERSION} VERSION_EQUAL 3.10) + include_guard() + endif() + + %(toplevel)s + + """) + + +data = {} +converter = Converter() + + +def GetDict(obj): + ret = {} + for k in dir(obj): + if not k.startswith("_"): + ret[k] = getattr(obj, k) + return ret + + +# We take the MODULE path as a command-line argument to ensure that we can find +# it regardless of how exactly Bazel was invoked. +exec(open(sys.argv[1]).read(), GetDict(ModuleFileFunctions(converter))) + +with open(sys.argv[2], "w") as f: + f.write(converter.convert()) diff --git a/cmake/gtest.cmake b/cmake/gtest.cmake index b891db9fe3..680ae7f265 100644 --- a/cmake/gtest.cmake +++ b/cmake/gtest.cmake @@ -2,6 +2,17 @@ option(protobuf_USE_EXTERNAL_GTEST "Use external Google Test (i.e. not the one i if (protobuf_USE_EXTERNAL_GTEST) find_package(GTest REQUIRED CONFIG) +elseif (protobuf_FETCH_DEPENDENCIES) + include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake) + include(FetchContent) + FetchContent_Declare( + googletest + GIT_REPOSITORY "https://github.com/google/googletest.git" + GIT_TAG "v${googletest-version}" + ) + # Due to https://github.com/google/googletest/issues/4384, we can't name this + # GTest for use with find_package until 1.15.0. + FetchContent_MakeAvailable(googletest) else() if (NOT EXISTS "${protobuf_SOURCE_DIR}/third_party/googletest/CMakeLists.txt") message(FATAL_ERROR diff --git a/regenerate_stale_files.sh b/regenerate_stale_files.sh index f5940b824d..b15c895144 100755 --- a/regenerate_stale_files.sh +++ b/regenerate_stale_files.sh @@ -15,6 +15,7 @@ readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS}" STALENESS_TESTS=( "java/core:generated_java_defaults_staleness_test" "upb/reflection:bootstrap_upb_defaults_staleness_test" + "cmake:test_dependencies_staleness" "src:cmake_lists_staleness_test" "src/google/protobuf:well_known_types_staleness_test" "objectivec:well_known_types_staleness_test" diff --git a/upb/cmake/BUILD.bazel b/upb/cmake/BUILD.bazel index 81338ed327..f5a9469d70 100644 --- a/upb/cmake/BUILD.bazel +++ b/upb/cmake/BUILD.bazel @@ -28,22 +28,6 @@ py_library( visibility = ["//visibility:public"], ) -py_binary( - name = "make_cmakelists", - srcs = ["make_cmakelists.py"], -) - -genrule( - name = "gen_cmakelists", - srcs = [ - "//upb:BUILD", - ], - outs = ["generated-in/CMakeLists.txt"], - cmd = "$(location :make_cmakelists) " + - "$(location //upb:BUILD) $@", - tools = [":make_cmakelists"], -) - genrule( name = "copy_protos", srcs = [ @@ -61,7 +45,6 @@ genrule( staleness_test( name = "test_generated_files", outs = [ - "CMakeLists.txt", "google/protobuf/descriptor.upb.h", "google/protobuf/descriptor.upb_minitable.c", "google/protobuf/descriptor.upb_minitable.h", @@ -70,49 +53,10 @@ staleness_test( tags = ["manual"], ) -# Test the CMake build ######################################################### - -make_shell_script( - name = "gen_run_cmake_build", - out = "run_cmake_build.sh", - contents = "set -ex\n" + - "cd $(dirname $1) && cp -r . .. && cd ../..\n" + - "mkdir build && cd build && cmake ../cmake && make -j8 && make test", -) - -sh_test( - name = "cmake_build", - srcs = ["run_cmake_build.sh"], - args = ["$(location :gen_cmakelists)"], - data = [ - ":copy_protos", - ":gen_cmakelists", - "//third_party/utf8_range:utf8_range_srcs", - "//upb:source_files", - "//upb/base:source_files", - "//upb/hash:source_files", - "//upb/lex:source_files", - "//upb/mem:source_files", - "//upb/message:source_files", - "//upb/mini_descriptor:source_files", - "//upb/mini_table:source_files", - "//upb/port:source_files", - "//upb/reflection:source_files", - "//upb/text:source_files", - "//upb/wire:source_files", - ], - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), - deps = ["@bazel_tools//tools/bash/runfiles"], -) - pkg_files( name = "upb_cmake_dist", srcs = [ ":copy_protos", - ":gen_cmakelists", "//third_party/utf8_range:utf8_range_srcs", "//upb:source_files", "//upb/base:source_files", diff --git a/upb/cmake/README.md b/upb/cmake/README.md deleted file mode 100644 index 7204207fd5..0000000000 --- a/upb/cmake/README.md +++ /dev/null @@ -1,23 +0,0 @@ - -# upb CMake build (EXPERIMENTAL) - -upb's CMake support is experimental. The core library builds successfully -under CMake, and this is verified by the Bazel tests in this directory. -However there is no support for building the upb compiler or for generating -.upb.c/upb.h files. This means upb's CMake support is incomplete at best, -unless your application is intended to be purely reflective. - -If you find this CMake setup useful in its current state, please consider -filing an issue so we know. If you have suggestions for how it could be -more useful (and particularly if you can contribute some code for it) -please feel free to file an issue for that too. Do keep in mind that upb -does not currently provide any ABI stability, so we want to avoid providing -a shared library. - -The CMakeLists.txt is generated from the Bazel BUILD files using the Python -scripts in this directory. We want to avoid having two separate sources of -truth that both need to be updated when a file is added or removed. - -This directory also contains some generated files that would be created -on the fly during a Bazel build. These are automatically kept in sync by -the Bazel test `//cmake:test_generated_files`. diff --git a/upb/cmake/make_cmakelists.py b/upb/cmake/make_cmakelists.py deleted file mode 100755 index 8135c6f205..0000000000 --- a/upb/cmake/make_cmakelists.py +++ /dev/null @@ -1,346 +0,0 @@ -#!/usr/bin/python -# -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""A tool to convert BUILD -> CMakeLists.txt. - -This tool is very upb-specific at the moment, and should not be seen as a -generic Bazel -> CMake converter. -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import sys -import textwrap -import os - -def StripFirstChar(deps): - return [dep[1:] for dep in deps] - -def IsSourceFile(name): - return name.endswith(".c") or name.endswith(".cc") - - -ADD_LIBRARY_FORMAT = """ -add_library(%(name)s %(type)s - %(sources)s -) -target_include_directories(%(name)s %(keyword)s - $ - $ - $ -) -""" - - -class BuildFileFunctions(object): - def __init__(self, converter): - self.converter = converter - - def _add_deps(self, kwargs, keyword=""): - if "deps" not in kwargs: - return - self.converter.toplevel += "target_link_libraries(%s%s\n %s)\n" % ( - kwargs["name"], - keyword, - "\n ".join(StripFirstChar(kwargs["deps"])) - ) - - def load(self, *args): - pass - - def cc_library(self, **kwargs): - if kwargs["name"].endswith("amalgamation"): - return - if kwargs["name"] == "upbc_generator": - return - if kwargs["name"] == "lupb": - return - if "testonly" in kwargs: - return - files = kwargs.get("srcs", []) + kwargs.get("hdrs", []) - found_files = [] - pregenerated_files = [ - "CMakeLists.txt", "descriptor.upb.h", "descriptor.upb.c" - ] - for file in files: - if os.path.basename(file) in pregenerated_files: - found_files.append("../cmake/" + file) - else: - found_files.append("../" + file) - - if list(filter(IsSourceFile, files)): - # Has sources, make this a normal library. - self.converter.toplevel += ADD_LIBRARY_FORMAT % { - "name": kwargs["name"], - "type": "", - "keyword": "PUBLIC", - "sources": "\n ".join(found_files), - } - self._add_deps(kwargs) - else: - # Header-only library, have to do a couple things differently. - # For some info, see: - # http://mariobadr.com/creating-a-header-only-library-with-cmake.html - self.converter.toplevel += ADD_LIBRARY_FORMAT % { - "name": kwargs["name"], - "type": "INTERFACE", - "keyword": "INTERFACE", - "sources": "", - } - self._add_deps(kwargs, " INTERFACE") - - def cc_binary(self, **kwargs): - pass - - def cc_test(self, **kwargs): - # Disable this until we properly support upb_proto_library(). - # self.converter.toplevel += "add_executable(%s\n %s)\n" % ( - # kwargs["name"], - # "\n ".join(kwargs["srcs"]) - # ) - # self.converter.toplevel += "add_test(NAME %s COMMAND %s)\n" % ( - # kwargs["name"], - # kwargs["name"], - # ) - - # if "data" in kwargs: - # for data_dep in kwargs["data"]: - # self.converter.toplevel += textwrap.dedent("""\ - # add_custom_command( - # TARGET %s POST_BUILD - # COMMAND ${CMAKE_COMMAND} -E copy - # ${CMAKE_SOURCE_DIR}/%s - # ${CMAKE_CURRENT_BINARY_DIR}/%s)\n""" % ( - # kwargs["name"], data_dep, data_dep - # )) - - # self._add_deps(kwargs) - pass - - def cc_fuzz_test(self, **kwargs): - pass - - def pkg_files(self, **kwargs): - pass - - def py_library(self, **kwargs): - pass - - def py_binary(self, **kwargs): - pass - - def lua_proto_library(self, **kwargs): - pass - - def sh_test(self, **kwargs): - pass - - def make_shell_script(self, **kwargs): - pass - - def exports_files(self, files, **kwargs): - pass - - def proto_library(self, **kwargs): - pass - - def cc_proto_library(self, **kwargs): - pass - - def staleness_test(self, **kwargs): - pass - - def upb_amalgamation(self, **kwargs): - pass - - def upb_proto_library(self, **kwargs): - pass - - def upb_minitable_proto_library(self, **kwargs): - pass - - def upb_proto_library_copts(self, **kwargs): - pass - - def upb_proto_reflection_library(self, **kwargs): - pass - - def upb_proto_srcs(self, **kwargs): - pass - - def genrule(self, **kwargs): - pass - - def config_setting(self, **kwargs): - pass - - def upb_fasttable_enabled(self, **kwargs): - pass - - def select(self, arg_dict): - return [] - - def glob(self, *args, **kwargs): - return [] - - def licenses(self, *args): - pass - - def filegroup(self, **kwargs): - pass - - def map_dep(self, arg): - return arg - - def package_group(self, **kwargs): - pass - - def bool_flag(self, **kwargs): - pass - - def bootstrap_upb_proto_library(self, **kwargs): - pass - - def bootstrap_cc_library(self, **kwargs): - pass - - def alias(self, **kwargs): - pass - - def package(self, **kwargs): - pass - -class Converter(object): - def __init__(self): - self.toplevel = "" - self.if_lua = "" - - def convert(self): - return self.template % { - "toplevel": converter.toplevel, - } - - template = textwrap.dedent("""\ - # This file was generated from BUILD using tools/make_cmakelists.py. - - cmake_minimum_required(VERSION 3.10...3.24) - - project(upb) - set(CMAKE_C_STANDARD 99) - - # Prevent CMake from setting -rdynamic on Linux (!!). - SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") - SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") - - # Set default build type. - if(NOT CMAKE_BUILD_TYPE) - message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.") - set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING - "Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel." - FORCE) - endif() - - # When using Ninja, compiler output won't be colorized without this. - include(CheckCXXCompilerFlag) - CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS) - if(SUPPORTS_COLOR_ALWAYS) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always") - endif() - - # Implement ASAN/UBSAN options - if(UPB_ENABLE_ASAN) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") - endif() - - if(UPB_ENABLE_UBSAN) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address") - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address") - endif() - - if(NOT TARGET utf8_range) - if(EXISTS ../../third_party/utf8_range) - # utf8_range is already installed - include_directories(../../third_party/utf8_range) - else() - include(FetchContent) - FetchContent_Declare( - utf8_range - GIT_REPOSITORY "https://github.com/protocolbuffers/utf8_range.git" - GIT_TAG "d863bc33e15cba6d873c878dcca9e6fe52b2f8cb" - ) - FetchContent_GetProperties(utf8_range) - if(NOT utf8_range_POPULATED) - FetchContent_Populate(utf8_range) - include_directories(${utf8_range_SOURCE_DIR}) - endif() - endif() - endif() - - if(APPLE) - set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace") - elseif(UNIX) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id") - endif() - - enable_testing() - - %(toplevel)s - - """) - -data = {} -converter = Converter() - -def GetDict(obj): - ret = {} - ret["UPB_DEFAULT_COPTS"] = [] # HACK - ret["UPB_DEFAULT_CPPOPTS"] = [] # HACK - for k in dir(obj): - if not k.startswith("_"): - ret[k] = getattr(obj, k); - return ret - -globs = GetDict(converter) - -# We take the BUILD path as a command-line argument to ensure that we can find -# it regardless of how exactly Bazel was invoked. -exec(open(sys.argv[1]).read(), GetDict(BuildFileFunctions(converter))) # BUILD - -with open(sys.argv[2], "w") as f: - f.write(converter.convert()) diff --git a/upb/cmake/push_auto_update.sh b/upb/cmake/push_auto_update.sh deleted file mode 100755 index 6b7dba00ad..0000000000 --- a/upb/cmake/push_auto_update.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# Protocol Buffers - Google's data interchange format -# Copyright 2023 Google LLC. All rights reserved. -# https://developers.google.com/protocol-buffers/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google LLC nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# This script updates checked-in generated files (currently CMakeLists.txt, -# descriptor.upb.h, and descriptor.upb.c), commits the resulting change, and -# pushes it. This does not do anything useful when run manually, but should be -# run by our GitHub action instead. - -set -ex - -# Exit early if the previous commit was made by the bot. This reduces the risk -# of a bug causing an infinite loop of auto-generated commits. -if (git log -1 --pretty=format:'%an' | grep -q "Protobuf Team Bot"); then - echo "Previous commit was authored by bot" - exit 0 -fi - -cd $(dirname -- "$0")/.. -bazel test //cmake:test_generated_files || bazel-bin/cmake/test_generated_files --fix - -# Try to determine the most recent pull request number. -title=$(git log -1 --pretty='%s') -pr_from_merge=$(echo "$title" | sed -n 's/^Merge pull request #\([0-9]\+\).*/\1/p') -pr_from_squash=$(echo "$title" | sed -n 's/^.*(#\([0-9]\+\))$/\1/p') - -pr_number="" -if [ ! -z "$pr_from_merge" ]; then - pr_number="$pr_from_merge" -elif [ ! -z "$pr_from_squash" ]; then - pr_number="$pr_from_squash" -fi - -if [ ! -z "$pr_number" ]; then - commit_message="Auto-generate CMake file lists after PR #$pr_number" -else - # If we are unable to determine the pull request number, we fall back on this - # default commit message. Typically this should not occur, but could happen - # if a pull request was merged via a rebase. - commit_message="Auto-generate CMake file lists" -fi - -git add -A -git diff --staged --quiet || git commit -am "$commit_message" -git push