Support fetching CMake dependencies via `-Dprotobuf_FETCH_DEPENDENCIES=ON`

This ports upb's WORKSPACE scraping logic to protobuf, and allows us to dynamically fetch our dependencies at the exact same pinned version as in Bazel via protobuf_FETCH_DEPENDENCIES=ON.  This is mostly for development purposes, and is preferable to git submodules.  In a later cl we will flip the default behavior to "package"

#test-continuous

PiperOrigin-RevId: 686265348
pull/18806/head
Mike Kruskal 5 months ago committed by Copybara-Service
parent 2b0a414573
commit d3735bc2a0
  1. 65
      .github/workflows/test_cpp.yml
  2. 55
      BUILD.bazel
  3. 18
      CMakeLists.txt
  4. 5
      MODULE.bazel
  5. 25
      cmake/BUILD.bazel
  6. 18
      cmake/abseil-cpp.cmake
  7. 30
      cmake/conformance.cmake
  8. 143
      cmake/dependencies_generator.py
  9. 11
      cmake/gtest.cmake
  10. 1
      regenerate_stale_files.sh
  11. 56
      upb/cmake/BUILD.bazel
  12. 23
      upb/cmake/README.md
  13. 346
      upb/cmake/make_cmakelists.py
  14. 73
      upb/cmake/push_auto_update.sh

@ -107,7 +107,6 @@ jobs:
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Cross compile protoc for ${{ matrix.arch }}
id: cross-compile
uses: protocolbuffers/protobuf-ci/cross-compile-protoc@v3
@ -131,7 +130,7 @@ jobs:
sccache -z;
cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
-Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=14
-Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
-Dprotobuf_WITH_ZLIB=OFF -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake --build . --parallel 20;
ctest --parallel 20;
sccache -s"
@ -141,18 +140,19 @@ jobs:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
include:
- flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
- flags: -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
- name: Ninja
flags: -G Ninja -DCMAKE_CXX_STANDARD=14
flags: -G Ninja -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
continuous-only: true
- name: Shared
flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14
flags: -Dprotobuf_BUILD_SHARED_LIBS=ON -Dprotobuf_BUILD_EXAMPLES=ON -DCMAKE_CXX_STANDARD=14 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
continuous-only: true
- name: C++17
flags: -DCMAKE_CXX_STANDARD=17
# TODO Re-enable this.
#- name: C++20
# flags: -DCMAKE_CXX_STANDARD=20
flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
- name: C++20
flags: -DCMAKE_CXX_STANDARD=20 -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
- name: Fetch
flags: -DCMAKE_CXX_STANDARD=17 -Dprotobuf_FETCH_DEPENDENCIES=ON
name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux CMake ${{ matrix.name}}
runs-on: ubuntu-latest
@ -174,37 +174,51 @@ jobs:
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
uses: protocolbuffers/protobuf-ci/docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/test.sh ${{ matrix.flags}} ${{ env.SCCACHE_CMAKE_FLAGS }}
-Dprotobuf_BUILD_TESTS=ON -Dprotobuf_USE_EXTERNAL_GTEST=ON
-Dprotobuf_ABSL_PROVIDER=package
-Dprotobuf_BUILD_TESTS=ON ${{ matrix.package_flags }}
linux-cmake-install:
name: Linux CMake Install
strategy:
fail-fast: false # Don't cancel all jobs if one fails.
matrix:
type: [package, fetch]
include:
# Set defaults
- type: package
name: Install
flags: -Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package -Dprotobuf_JSONCPP_PROVIDER=package
- type: fetch
name: Install (Fetch)
flags: -Dprotobuf_FETCH_DEPENDENCIES=ON
continuous-only: true
name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }}Linux CMake ${{ matrix.name }})
runs-on: ubuntu-latest
steps:
- name: Checkout pending changes
uses: protocolbuffers/protobuf-ci/checkout@v3
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
with:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@v3
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
with:
cache-prefix: linux-cmake-install
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v3
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
-Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
${{ matrix.flags }}
-Dprotobuf_BUILD_SHARED_LIBS=ON \&\&
/test.sh
${{ env.SCCACHE_CMAKE_FLAGS }}
@ -212,7 +226,7 @@ jobs:
-Dprotobuf_BUILD_PROTOBUF_BINARIES=OFF
-Dprotobuf_BUILD_CONFORMANCE=ON
-DCMAKE_CXX_STANDARD=14
-Dprotobuf_USE_EXTERNAL_GTEST=ON -Dprotobuf_ABSL_PROVIDER=package
${{ matrix.flags }}
# This test should always be skipped on presubmit
linux-cmake-examples:
@ -236,7 +250,7 @@ jobs:
if: ${{ inputs.continuous-run }}
uses: protocolbuffers/protobuf-ci/docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/install.sh -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }}
@ -268,7 +282,6 @@ jobs:
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup sccache
if: ${{ !matrix.continuous-only || inputs.continuous-run }}
@ -288,7 +301,7 @@ jobs:
-c 'set -ex;
cd /workspace;
sccache -z;
cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_FETCH_DEPENDENCIES=ON;
cmake --build . --parallel 20;
ctest --verbose --parallel 20;
sccache -s'
@ -312,7 +325,7 @@ jobs:
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.13.3-384d5abe83a791c6b1ce04f5d7bc0b1f84a30d38
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/cmake:3.16.9-f39fc8b4e244fe5cd4c7138d0b6959a52b46ca48
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/test.sh ${{ env.SCCACHE_CMAKE_FLAGS }}
@ -326,7 +339,6 @@ jobs:
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup sccache
uses: protocolbuffers/protobuf-ci/sccache@v3
@ -337,14 +349,14 @@ jobs:
- name: Run tests
uses: protocolbuffers/protobuf-ci/docker@v3
with:
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:429f924aec315704b4233adcbe4b29006116f27769db98acd176b9eb69c31299
image: us-docker.pkg.dev/protobuf-build/containers/test/linux/32bit@sha256:56548bef786201330017eae685cc3d2fdb564fd2ca3b88e30e28d84572e4c5dd
platform: linux/386
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: >-
/bin/bash -cex '
cd /workspace;
sccache -z;
cmake . -DCMAKE_CXX_STANDARD=14 ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake . -DCMAKE_CXX_STANDARD=14 -Dprotobuf_FETCH_DEPENDENCIES=ON ${{ env.SCCACHE_CMAKE_FLAGS }};
cmake --build . --parallel 20;
ctest --verbose --parallel 20;
sccache -s'
@ -460,7 +472,6 @@ jobs:
uses: protocolbuffers/protobuf-ci/checkout@v3
with:
ref: ${{ inputs.safe-checkout }}
submodules: recursive
- name: Setup MSVC
if: ${{ runner.os == 'Windows' && (!matrix.continuous-only || inputs.continuous-run) }}
@ -499,7 +510,7 @@ jobs:
uses: protocolbuffers/protobuf-ci/bash@v3
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
command: cmake . ${{ matrix.install-flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON
- name: Build for install
if: ${{ matrix.install-flags && (!matrix.continuous-only || inputs.continuous-run) }}
shell: bash
@ -522,7 +533,7 @@ jobs:
uses: protocolbuffers/protobuf-ci/bash@v3
with:
credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON
command: cmake . ${{ matrix.flags }} ${{ env.SCCACHE_CMAKE_FLAGS }} -Dprotobuf_ALLOW_CCACHE=ON -Dprotobuf_FETCH_DEPENDENCIES=ON
- name: Build
if: ${{ !matrix.continuous-only || inputs.continuous-run }}

@ -24,6 +24,11 @@ license(
license_text = ":LICENSE",
)
exports_files(
["MODULE.bazel"],
visibility = ["//cmake:__pkg__"],
)
################################################################################
# Well Known Types Proto Library Rules
#
@ -642,53 +647,3 @@ filegroup(
srcs = glob(["**/*.bzl"]),
visibility = ["//visibility:public"],
)
################################################################################
# Packaging rules
################################################################################
# Files included in all source distributions
pkg_files(
name = "common_dist_files",
srcs = glob(
[
"*.bzl",
"cmake/*.cmake",
"cmake/*.in",
"editors/*",
],
allow_empty = True,
) + [
"BUILD.bazel",
"CMakeLists.txt",
"CONTRIBUTORS.txt",
"LICENSE",
"README.md",
"WORKSPACE",
"cmake/README.md",
"generate_descriptor_proto.sh",
"maven_install.json",
"//third_party:BUILD.bazel",
"//third_party:zlib.BUILD",
],
strip_prefix = strip_prefix.from_root(""),
visibility = ["//pkg:__pkg__"],
)
# Additional files for C#
pkg_files(
name = "csharp_dist_files",
srcs = [
"global.json",
],
visibility = ["//pkg:__pkg__"],
)
# Additional files for ObjC
pkg_files(
name = "objectivec_dist_files",
srcs = [
"Protobuf.podspec",
],
visibility = ["//pkg:__pkg__"],
)

@ -153,14 +153,6 @@ file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/cmaketest.map)
find_package(Threads REQUIRED)
# We can install dependencies from submodules if we're running
# CMake v3.13 or newer.
if(CMAKE_VERSION VERSION_LESS 3.13)
set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE OFF)
else()
set(_protobuf_INSTALL_SUPPORTED_FROM_MODULE ON)
endif()
set(_protobuf_FIND_ZLIB)
if (protobuf_WITH_ZLIB)
find_package(ZLIB)
@ -289,11 +281,13 @@ include_directories(
${protobuf_BINARY_DIR}/src
${protobuf_SOURCE_DIR}/src)
set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library")
set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package")
set(protobuf_FETCH_DEPENDENCIES OFF CACHE BOOL "Download dependencies from GitHub. If this option is not set, the dependency must be available locally, either as a sub-module or an installed package.")
set(protobuf_ABSL_PROVIDER "module" CACHE STRING "Provider of absl library. `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub")
set_property(CACHE protobuf_ABSL_PROVIDER PROPERTY STRINGS "module" "package" "fetch")
set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library")
set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package")
set(protobuf_JSONCPP_PROVIDER "module" CACHE STRING "Provider of jsoncpp library. `module` uses sub-modules, `package` searches for a local installation, and `fetch` downloads from GitHub")
set_property(CACHE protobuf_JSONCPP_PROVIDER PROPERTY STRINGS "module" "package" "fetch")
if (protobuf_BUILD_TESTS)
include(${protobuf_SOURCE_DIR}/cmake/gtest.cmake)

@ -12,7 +12,7 @@ module(
# Bzlmod follows MVS:
# https://bazel.build/versions/6.0.0/build/bzlmod#version-resolution
# Thus the highest version in their module graph is resolved.
bazel_dep(name = "abseil-cpp", version = "20230802.0.bcr.1", repo_name = "com_google_absl")
bazel_dep(name = "abseil-cpp", version = "20230802.1", repo_name = "com_google_absl")
bazel_dep(name = "bazel_skylib", version = "1.7.0")
bazel_dep(name = "jsoncpp", version = "1.9.5")
bazel_dep(name = "rules_cc", version = "0.0.13")
@ -27,10 +27,9 @@ bazel_dep(name = "rules_rust", version = "0.51.0")
bazel_dep(name = "platforms", version = "0.0.8")
bazel_dep(name = "zlib", version = "1.3.1")
bazel_dep(name = "bazel_features", version = "1.17.0", repo_name = "proto_bazel_features")
bazel_dep(
name = "rules_shell",
version = "0.2.0"
version = "0.2.0",
)
# Proto toolchains

@ -0,0 +1,25 @@
load("@rules_python//python:defs.bzl", "py_binary")
load("//upb/cmake:build_defs.bzl", "staleness_test")
py_binary(
name = "dependencies_generator",
srcs = ["dependencies_generator.py"],
)
genrule(
name = "generate_dependencies",
srcs = ["//:MODULE.bazel"],
outs = ["generated-in/dependencies.cmake"],
cmd = "$(location :dependencies_generator) " +
"$(location //:MODULE.bazel) $@",
tools = [":dependencies_generator"],
)
staleness_test(
name = "test_dependencies_staleness",
outs = [
"dependencies.cmake",
],
generated_pattern = "generated-in/%s",
tags = ["manual"],
)

@ -13,6 +13,20 @@ endif()
if(TARGET absl::strings)
# If Abseil is included already, skip including it.
# (https://github.com/protocolbuffers/protobuf/issues/10435)
elseif (protobuf_FETCH_DEPENDENCIES OR protobuf_ABSL_PROVIDER STREQUAL "fetch")
include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake)
include(FetchContent)
FetchContent_Declare(
absl
GIT_REPOSITORY "https://github.com/abseil/abseil-cpp.git"
GIT_TAG "${abseil-cpp-version}"
)
if(protobuf_INSTALL)
# When protobuf_INSTALL is enabled and Abseil will be built as a module,
# Abseil will be installed along with protobuf for convenience.
set(ABSL_ENABLE_INSTALL ON)
endif()
FetchContent_MakeAvailable(absl)
elseif(protobuf_ABSL_PROVIDER STREQUAL "module")
if(NOT ABSL_ROOT_DIR)
set(ABSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/abseil-cpp)
@ -27,10 +41,6 @@ elseif(protobuf_ABSL_PROVIDER STREQUAL "module")
else()
message(WARNING "protobuf_ABSL_PROVIDER is \"module\" but ABSL_ROOT_DIR is wrong")
endif()
if(protobuf_INSTALL AND NOT _protobuf_INSTALL_SUPPORTED_FROM_MODULE)
message(WARNING "protobuf_INSTALL will be forced to FALSE because protobuf_ABSL_PROVIDER is \"module\" and CMake version (${CMAKE_VERSION}) is less than 3.13.")
set(protobuf_INSTALL FALSE)
endif()
elseif(protobuf_ABSL_PROVIDER STREQUAL "package")
# Use "CONFIG" as there is no built-in cmake module for absl.
find_package(absl REQUIRED CONFIG)

@ -1,4 +1,17 @@
if (protobuf_JSONCPP_PROVIDER STREQUAL "module")
# Don't run jsoncpp tests.
set(JSONCPP_WITH_TESTS OFF)
if (protobuf_FETCH_DEPENDENCIES OR protobuf_JSONCPP_PROVIDER STREQUAL "fetch")
include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake)
include(FetchContent)
FetchContent_Declare(
jsoncpp
GIT_REPOSITORY "https://github.com/open-source-parsers/jsoncpp.git"
# TODO Use ${jsoncpp-version} here once it supports cmake.
GIT_TAG "1.9.4"
)
FetchContent_MakeAvailable(jsoncpp)
elseif (protobuf_JSONCPP_PROVIDER STREQUAL "module")
if (NOT EXISTS "${protobuf_SOURCE_DIR}/third_party/jsoncpp/CMakeLists.txt")
message(FATAL_ERROR
"Cannot find third_party/jsoncpp directory that's needed to "
@ -129,18 +142,17 @@ add_test(NAME conformance_cpp_test
--text_format_failure_list ${protobuf_SOURCE_DIR}/conformance/text_format_failure_list_cpp.txt
--output_dir ${protobuf_TEST_XML_OUTDIR}
--maximum_edition 2023
${CMAKE_CURRENT_BINARY_DIR}/conformance_cpp
$<TARGET_FILE:conformance_cpp>
DEPENDS conformance_test_runner conformance_cpp)
set(JSONCPP_WITH_TESTS OFF CACHE BOOL "Disable tests")
if(protobuf_JSONCPP_PROVIDER STREQUAL "module")
if(NOT protobuf_FETCH_DEPENDENCIES AND protobuf_JSONCPP_PROVIDER STREQUAL "module")
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/third_party/jsoncpp third_party/jsoncpp)
target_include_directories(conformance_test_runner PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/third_party/jsoncpp/include)
if(BUILD_SHARED_LIBS)
target_link_libraries(conformance_test_runner jsoncpp_lib)
else()
target_link_libraries(conformance_test_runner jsoncpp_static)
endif()
endif()
if(BUILD_SHARED_LIBS)
target_link_libraries(conformance_test_runner jsoncpp_lib)
else()
target_link_libraries(conformance_test_runner jsoncpp)
target_link_libraries(conformance_test_runner jsoncpp_static)
endif()

@ -0,0 +1,143 @@
#!/usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google LLC. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google LLC nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A tool to convert MODULE.bazel -> CMakeLists.txt.
This tool is very protobuf-specific at the moment, and should not be seen as a
generic Bazel -> CMake converter.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import sys
import textwrap
class ExtensionFunctions(object):
"""A fake extension that we can use to get the functions we need."""
def toolchain(self, *args, **kwargs):
pass
def parse(self, *args, **kwargs):
pass
def spec(self, *args, **kwargs):
pass
def from_specs(self, *args, **kwargs):
pass
def install(self, *args, **kwargs):
pass
class ModuleFileFunctions(object):
"""A fake MODULE file that we can exec() to get the functions we need."""
def __init__(self, converter):
self.converter = converter
def module(self, *args, **kwargs):
pass
def bazel_dep(self, name, version, **kwargs):
self.converter.toplevel += textwrap.dedent(
"""\
set(%(name)s-version "%(version)s")
"""
% {
"name": name,
"version": version,
}
)
def register_toolchains(self, *args):
pass
def use_repo(self, *args, **kwargs):
pass
def use_extension(self, *args):
return ExtensionFunctions()
class Converter(object):
def __init__(self):
self.toplevel = ""
self.if_lua = ""
def convert(self):
return self.template % {
"toplevel": converter.toplevel,
}
template = textwrap.dedent("""\
# Auto-generated by @//cmake:make_dependencies
#
# This file contains lists of external dependencies based on our Bazel
# config. It should be included from a hand-written CMake file that uses
# them.
#
# Changes to this file will be overwritten based on Bazel definitions.
if(${CMAKE_VERSION} VERSION_GREATER 3.10 OR ${CMAKE_VERSION} VERSION_EQUAL 3.10)
include_guard()
endif()
%(toplevel)s
""")
data = {}
converter = Converter()
def GetDict(obj):
ret = {}
for k in dir(obj):
if not k.startswith("_"):
ret[k] = getattr(obj, k)
return ret
# We take the MODULE path as a command-line argument to ensure that we can find
# it regardless of how exactly Bazel was invoked.
exec(open(sys.argv[1]).read(), GetDict(ModuleFileFunctions(converter)))
with open(sys.argv[2], "w") as f:
f.write(converter.convert())

@ -2,6 +2,17 @@ option(protobuf_USE_EXTERNAL_GTEST "Use external Google Test (i.e. not the one i
if (protobuf_USE_EXTERNAL_GTEST)
find_package(GTest REQUIRED CONFIG)
elseif (protobuf_FETCH_DEPENDENCIES)
include(${protobuf_SOURCE_DIR}/cmake/dependencies.cmake)
include(FetchContent)
FetchContent_Declare(
googletest
GIT_REPOSITORY "https://github.com/google/googletest.git"
GIT_TAG "v${googletest-version}"
)
# Due to https://github.com/google/googletest/issues/4384, we can't name this
# GTest for use with find_package until 1.15.0.
FetchContent_MakeAvailable(googletest)
else()
if (NOT EXISTS "${protobuf_SOURCE_DIR}/third_party/googletest/CMakeLists.txt")
message(FATAL_ERROR

@ -15,6 +15,7 @@ readonly BazelBin="${BAZEL:-bazel} ${BAZEL_STARTUP_FLAGS}"
STALENESS_TESTS=(
"java/core:generated_java_defaults_staleness_test"
"upb/reflection:bootstrap_upb_defaults_staleness_test"
"cmake:test_dependencies_staleness"
"src:cmake_lists_staleness_test"
"src/google/protobuf:well_known_types_staleness_test"
"objectivec:well_known_types_staleness_test"

@ -28,22 +28,6 @@ py_library(
visibility = ["//visibility:public"],
)
py_binary(
name = "make_cmakelists",
srcs = ["make_cmakelists.py"],
)
genrule(
name = "gen_cmakelists",
srcs = [
"//upb:BUILD",
],
outs = ["generated-in/CMakeLists.txt"],
cmd = "$(location :make_cmakelists) " +
"$(location //upb:BUILD) $@",
tools = [":make_cmakelists"],
)
genrule(
name = "copy_protos",
srcs = [
@ -61,7 +45,6 @@ genrule(
staleness_test(
name = "test_generated_files",
outs = [
"CMakeLists.txt",
"google/protobuf/descriptor.upb.h",
"google/protobuf/descriptor.upb_minitable.c",
"google/protobuf/descriptor.upb_minitable.h",
@ -70,49 +53,10 @@ staleness_test(
tags = ["manual"],
)
# Test the CMake build #########################################################
make_shell_script(
name = "gen_run_cmake_build",
out = "run_cmake_build.sh",
contents = "set -ex\n" +
"cd $(dirname $1) && cp -r . .. && cd ../..\n" +
"mkdir build && cd build && cmake ../cmake && make -j8 && make test",
)
sh_test(
name = "cmake_build",
srcs = ["run_cmake_build.sh"],
args = ["$(location :gen_cmakelists)"],
data = [
":copy_protos",
":gen_cmakelists",
"//third_party/utf8_range:utf8_range_srcs",
"//upb:source_files",
"//upb/base:source_files",
"//upb/hash:source_files",
"//upb/lex:source_files",
"//upb/mem:source_files",
"//upb/message:source_files",
"//upb/mini_descriptor:source_files",
"//upb/mini_table:source_files",
"//upb/port:source_files",
"//upb/reflection:source_files",
"//upb/text:source_files",
"//upb/wire:source_files",
],
target_compatible_with = select({
"@platforms//os:windows": ["@platforms//:incompatible"],
"//conditions:default": [],
}),
deps = ["@bazel_tools//tools/bash/runfiles"],
)
pkg_files(
name = "upb_cmake_dist",
srcs = [
":copy_protos",
":gen_cmakelists",
"//third_party/utf8_range:utf8_range_srcs",
"//upb:source_files",
"//upb/base:source_files",

@ -1,23 +0,0 @@
# upb CMake build (EXPERIMENTAL)
upb's CMake support is experimental. The core library builds successfully
under CMake, and this is verified by the Bazel tests in this directory.
However there is no support for building the upb compiler or for generating
.upb.c/upb.h files. This means upb's CMake support is incomplete at best,
unless your application is intended to be purely reflective.
If you find this CMake setup useful in its current state, please consider
filing an issue so we know. If you have suggestions for how it could be
more useful (and particularly if you can contribute some code for it)
please feel free to file an issue for that too. Do keep in mind that upb
does not currently provide any ABI stability, so we want to avoid providing
a shared library.
The CMakeLists.txt is generated from the Bazel BUILD files using the Python
scripts in this directory. We want to avoid having two separate sources of
truth that both need to be updated when a file is added or removed.
This directory also contains some generated files that would be created
on the fly during a Bazel build. These are automatically kept in sync by
the Bazel test `//cmake:test_generated_files`.

@ -1,346 +0,0 @@
#!/usr/bin/python
#
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google LLC. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google LLC nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""A tool to convert BUILD -> CMakeLists.txt.
This tool is very upb-specific at the moment, and should not be seen as a
generic Bazel -> CMake converter.
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
import textwrap
import os
def StripFirstChar(deps):
return [dep[1:] for dep in deps]
def IsSourceFile(name):
return name.endswith(".c") or name.endswith(".cc")
ADD_LIBRARY_FORMAT = """
add_library(%(name)s %(type)s
%(sources)s
)
target_include_directories(%(name)s %(keyword)s
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/..>
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../cmake>
$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
)
"""
class BuildFileFunctions(object):
def __init__(self, converter):
self.converter = converter
def _add_deps(self, kwargs, keyword=""):
if "deps" not in kwargs:
return
self.converter.toplevel += "target_link_libraries(%s%s\n %s)\n" % (
kwargs["name"],
keyword,
"\n ".join(StripFirstChar(kwargs["deps"]))
)
def load(self, *args):
pass
def cc_library(self, **kwargs):
if kwargs["name"].endswith("amalgamation"):
return
if kwargs["name"] == "upbc_generator":
return
if kwargs["name"] == "lupb":
return
if "testonly" in kwargs:
return
files = kwargs.get("srcs", []) + kwargs.get("hdrs", [])
found_files = []
pregenerated_files = [
"CMakeLists.txt", "descriptor.upb.h", "descriptor.upb.c"
]
for file in files:
if os.path.basename(file) in pregenerated_files:
found_files.append("../cmake/" + file)
else:
found_files.append("../" + file)
if list(filter(IsSourceFile, files)):
# Has sources, make this a normal library.
self.converter.toplevel += ADD_LIBRARY_FORMAT % {
"name": kwargs["name"],
"type": "",
"keyword": "PUBLIC",
"sources": "\n ".join(found_files),
}
self._add_deps(kwargs)
else:
# Header-only library, have to do a couple things differently.
# For some info, see:
# http://mariobadr.com/creating-a-header-only-library-with-cmake.html
self.converter.toplevel += ADD_LIBRARY_FORMAT % {
"name": kwargs["name"],
"type": "INTERFACE",
"keyword": "INTERFACE",
"sources": "",
}
self._add_deps(kwargs, " INTERFACE")
def cc_binary(self, **kwargs):
pass
def cc_test(self, **kwargs):
# Disable this until we properly support upb_proto_library().
# self.converter.toplevel += "add_executable(%s\n %s)\n" % (
# kwargs["name"],
# "\n ".join(kwargs["srcs"])
# )
# self.converter.toplevel += "add_test(NAME %s COMMAND %s)\n" % (
# kwargs["name"],
# kwargs["name"],
# )
# if "data" in kwargs:
# for data_dep in kwargs["data"]:
# self.converter.toplevel += textwrap.dedent("""\
# add_custom_command(
# TARGET %s POST_BUILD
# COMMAND ${CMAKE_COMMAND} -E copy
# ${CMAKE_SOURCE_DIR}/%s
# ${CMAKE_CURRENT_BINARY_DIR}/%s)\n""" % (
# kwargs["name"], data_dep, data_dep
# ))
# self._add_deps(kwargs)
pass
def cc_fuzz_test(self, **kwargs):
pass
def pkg_files(self, **kwargs):
pass
def py_library(self, **kwargs):
pass
def py_binary(self, **kwargs):
pass
def lua_proto_library(self, **kwargs):
pass
def sh_test(self, **kwargs):
pass
def make_shell_script(self, **kwargs):
pass
def exports_files(self, files, **kwargs):
pass
def proto_library(self, **kwargs):
pass
def cc_proto_library(self, **kwargs):
pass
def staleness_test(self, **kwargs):
pass
def upb_amalgamation(self, **kwargs):
pass
def upb_proto_library(self, **kwargs):
pass
def upb_minitable_proto_library(self, **kwargs):
pass
def upb_proto_library_copts(self, **kwargs):
pass
def upb_proto_reflection_library(self, **kwargs):
pass
def upb_proto_srcs(self, **kwargs):
pass
def genrule(self, **kwargs):
pass
def config_setting(self, **kwargs):
pass
def upb_fasttable_enabled(self, **kwargs):
pass
def select(self, arg_dict):
return []
def glob(self, *args, **kwargs):
return []
def licenses(self, *args):
pass
def filegroup(self, **kwargs):
pass
def map_dep(self, arg):
return arg
def package_group(self, **kwargs):
pass
def bool_flag(self, **kwargs):
pass
def bootstrap_upb_proto_library(self, **kwargs):
pass
def bootstrap_cc_library(self, **kwargs):
pass
def alias(self, **kwargs):
pass
def package(self, **kwargs):
pass
class Converter(object):
def __init__(self):
self.toplevel = ""
self.if_lua = ""
def convert(self):
return self.template % {
"toplevel": converter.toplevel,
}
template = textwrap.dedent("""\
# This file was generated from BUILD using tools/make_cmakelists.py.
cmake_minimum_required(VERSION 3.10...3.24)
project(upb)
set(CMAKE_C_STANDARD 99)
# Prevent CMake from setting -rdynamic on Linux (!!).
SET(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "")
SET(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "")
# Set default build type.
if(NOT CMAKE_BUILD_TYPE)
message(STATUS "Setting build type to 'RelWithDebInfo' as none was specified.")
set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING
"Choose the type of build, options are: Debug Release RelWithDebInfo MinSizeRel."
FORCE)
endif()
# When using Ninja, compiler output won't be colorized without this.
include(CheckCXXCompilerFlag)
CHECK_CXX_COMPILER_FLAG(-fdiagnostics-color=always SUPPORTS_COLOR_ALWAYS)
if(SUPPORTS_COLOR_ALWAYS)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
endif()
# Implement ASAN/UBSAN options
if(UPB_ENABLE_ASAN)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address")
endif()
if(UPB_ENABLE_UBSAN)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fsanitize=address")
endif()
if(NOT TARGET utf8_range)
if(EXISTS ../../third_party/utf8_range)
# utf8_range is already installed
include_directories(../../third_party/utf8_range)
else()
include(FetchContent)
FetchContent_Declare(
utf8_range
GIT_REPOSITORY "https://github.com/protocolbuffers/utf8_range.git"
GIT_TAG "d863bc33e15cba6d873c878dcca9e6fe52b2f8cb"
)
FetchContent_GetProperties(utf8_range)
if(NOT utf8_range_POPULATED)
FetchContent_Populate(utf8_range)
include_directories(${utf8_range_SOURCE_DIR})
endif()
endif()
endif()
if(APPLE)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -undefined dynamic_lookup -flat_namespace")
elseif(UNIX)
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--build-id")
endif()
enable_testing()
%(toplevel)s
""")
data = {}
converter = Converter()
def GetDict(obj):
ret = {}
ret["UPB_DEFAULT_COPTS"] = [] # HACK
ret["UPB_DEFAULT_CPPOPTS"] = [] # HACK
for k in dir(obj):
if not k.startswith("_"):
ret[k] = getattr(obj, k);
return ret
globs = GetDict(converter)
# We take the BUILD path as a command-line argument to ensure that we can find
# it regardless of how exactly Bazel was invoked.
exec(open(sys.argv[1]).read(), GetDict(BuildFileFunctions(converter))) # BUILD
with open(sys.argv[2], "w") as f:
f.write(converter.convert())

@ -1,73 +0,0 @@
#!/bin/bash
# Protocol Buffers - Google's data interchange format
# Copyright 2023 Google LLC. All rights reserved.
# https://developers.google.com/protocol-buffers/
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google LLC nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# This script updates checked-in generated files (currently CMakeLists.txt,
# descriptor.upb.h, and descriptor.upb.c), commits the resulting change, and
# pushes it. This does not do anything useful when run manually, but should be
# run by our GitHub action instead.
set -ex
# Exit early if the previous commit was made by the bot. This reduces the risk
# of a bug causing an infinite loop of auto-generated commits.
if (git log -1 --pretty=format:'%an' | grep -q "Protobuf Team Bot"); then
echo "Previous commit was authored by bot"
exit 0
fi
cd $(dirname -- "$0")/..
bazel test //cmake:test_generated_files || bazel-bin/cmake/test_generated_files --fix
# Try to determine the most recent pull request number.
title=$(git log -1 --pretty='%s')
pr_from_merge=$(echo "$title" | sed -n 's/^Merge pull request #\([0-9]\+\).*/\1/p')
pr_from_squash=$(echo "$title" | sed -n 's/^.*(#\([0-9]\+\))$/\1/p')
pr_number=""
if [ ! -z "$pr_from_merge" ]; then
pr_number="$pr_from_merge"
elif [ ! -z "$pr_from_squash" ]; then
pr_number="$pr_from_squash"
fi
if [ ! -z "$pr_number" ]; then
commit_message="Auto-generate CMake file lists after PR #$pr_number"
else
# If we are unable to determine the pull request number, we fall back on this
# default commit message. Typically this should not occur, but could happen
# if a pull request was merged via a rebase.
commit_message="Auto-generate CMake file lists"
fi
git add -A
git diff --staged --quiet || git commit -am "$commit_message"
git push
Loading…
Cancel
Save