[xDS Proto] Enhence gRPC buildgen for 3rd party proto compilation (#29254)

* [xDS Proto] Enhence gRPC buildgen for 3rd party proto compilation

* Rebase from master to update envoy-api version

* Address reviewer's comments

* Address reviewer's comment

* Regenerate project

* Rename external_library

* Address reviewer's comments

* Add comments for the internals of generate C++ proto code

* Add proto file as a dependency to the custom command
pull/29290/head
Lidi Zheng 3 years ago committed by GitHub
parent 61987ec3a2
commit 99752b173c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 435
      CMakeLists.txt
  2. 29
      build_autogenerated.yaml
  3. 40
      cmake/download_archive.cmake
  4. 154
      templates/CMakeLists.txt.template
  5. 6
      test/distrib/cpp/run_distrib_test_cmake.bat
  6. 151
      tools/buildgen/extract_metadata_from_bazel_xml.py
  7. 17
      tools/buildgen/plugins/check_attrs.py

435
CMakeLists.txt generated

@ -318,11 +318,92 @@ include(cmake/ssl.cmake)
include(cmake/upb.cmake)
include(cmake/xxhash.cmake)
include(cmake/zlib.cmake)
include(cmake/download_archive.cmake)
# Setup external proto library at third_party/envoy-api with 2 download URLs
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api)
# Download the archive via HTTP, validate the checksum, and extract to third_party/envoy-api.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api
https://storage.googleapis.com/grpc-bazel-mirror/github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz
c5807010b67033330915ca5a20483e30538ae5e689aa14b3631d6284beca4630
data-plane-api-9c42588c956220b48eb3099d186487c2f04d32ec
)
endif()
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api)
# Download the archive via HTTP, validate the checksum, and extract to third_party/envoy-api.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api
https://github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz
c5807010b67033330915ca5a20483e30538ae5e689aa14b3631d6284beca4630
data-plane-api-9c42588c956220b48eb3099d186487c2f04d32ec
)
endif()
# Setup external proto library at third_party/googleapis with 2 download URLs
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis)
# Download the archive via HTTP, validate the checksum, and extract to third_party/googleapis.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis
https://storage.googleapis.com/grpc-bazel-mirror/github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz
5bb6b0253ccf64b53d6c7249625a7e3f6c3bc6402abd52d3778bfa48258703a0
googleapis-2f9af297c84c55c8b871ba4495e01ade42476c92
)
endif()
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis)
# Download the archive via HTTP, validate the checksum, and extract to third_party/googleapis.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis
https://github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz
5bb6b0253ccf64b53d6c7249625a7e3f6c3bc6402abd52d3778bfa48258703a0
googleapis-2f9af297c84c55c8b871ba4495e01ade42476c92
)
endif()
# Setup external proto library at third_party/opencensus-proto/src with 2 download URLs
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src)
# Download the archive via HTTP, validate the checksum, and extract to third_party/opencensus-proto/src.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src
https://storage.googleapis.com/grpc-bazel-mirror/github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz
b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0
opencensus-proto-0.3.0/src
)
endif()
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src)
# Download the archive via HTTP, validate the checksum, and extract to third_party/opencensus-proto/src.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src
https://github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz
b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0
opencensus-proto-0.3.0/src
)
endif()
# Setup external proto library at third_party/xds with 2 download URLs
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds)
# Download the archive via HTTP, validate the checksum, and extract to third_party/xds.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds
https://storage.googleapis.com/grpc-bazel-mirror/github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz
5bc8365613fe2f8ce6cc33959b7667b13b7fe56cb9d16ba740c06e1a7c4242fc
xds-cb28da3451f158a947dfc45090fe92b07b243bc1
)
endif()
if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds)
# Download the archive via HTTP, validate the checksum, and extract to third_party/xds.
download_archive(
${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds
https://github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz
5bc8365613fe2f8ce6cc33959b7667b13b7fe56cb9d16ba740c06e1a7c4242fc
xds-cb28da3451f158a947dfc45090fe92b07b243bc1
)
endif()
if(WIN32)
set(_gRPC_BASELIB_LIBRARIES ws2_32 crypt32)
endif()
# Create directory for proto source files
set(_gRPC_PROTO_SRCS_DIR ${CMAKE_BINARY_DIR}/protos)
file(MAKE_DIRECTORY ${_gRPC_PROTO_SRCS_DIR})
# Create directory for generated .proto files
set(_gRPC_PROTO_GENS_DIR ${CMAKE_BINARY_DIR}/gens)
file(MAKE_DIRECTORY ${_gRPC_PROTO_GENS_DIR})
@ -330,6 +411,10 @@ file(MAKE_DIRECTORY ${_gRPC_PROTO_GENS_DIR})
# protobuf_generate_grpc_cpp
# --------------------------
#
# This method is no longer used by gRPC's CMake build process. However, it
# is used by many open source dependencies, that we might want to keep
# backward compatibility here.
#
# Add custom commands to process ``.proto`` files to C++ using protoc and
# GRPC plugin::
#
@ -344,40 +429,90 @@ function(protobuf_generate_grpc_cpp)
return()
endif()
set(_protobuf_include_path -I . -I ${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR})
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
file(RELATIVE_PATH REL_FIL ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL})
get_filename_component(REL_DIR ${REL_FIL} DIRECTORY)
set(RELFIL_WE "${REL_DIR}/${FIL_WE}")
#if cross-compiling, find host plugin
if(CMAKE_CROSSCOMPILING)
find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin)
else()
set(_gRPC_CPP_PLUGIN $<TARGET_FILE:grpc_cpp_plugin>)
endif()
add_custom_command(
OUTPUT "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h"
COMMAND ${_gRPC_PROTOBUF_PROTOC_EXECUTABLE}
ARGS --grpc_out=generate_mock_code=true:${_gRPC_PROTO_GENS_DIR}
--cpp_out=${_gRPC_PROTO_GENS_DIR}
--plugin=protoc-gen-grpc=${_gRPC_CPP_PLUGIN}
${_protobuf_include_path}
${REL_FIL}
DEPENDS ${ABS_FIL} ${_gRPC_PROTOBUF_PROTOC} ${_gRPC_CPP_PLUGIN}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}"
VERBATIM)
protobuf_generate_grpc_cpp_with_import_path_correction(${FIL} ${FIL})
endforeach()
endfunction()
# protobuf_generate_grpc_cpp_with_import_path_correction
# --------------------------
#
# Add custom commands to process ``.proto`` files to C++ using protoc and
# GRPC plugin::
#
# protobuf_generate_grpc_cpp_with_import_path_correction <FILE_LOCATION> <IMPORT_PATH>
#
# ``FILE_LOCATION``
# The relative path of the ``.proto`` file to the project root
# ``IMPORT_PATH``
# The proto import path that itself expected to be placed in. For
# example, a "bar.proto" file wants to be imported as
# `import "foo/bar.proto"`. Then we should place it under
# "<ProtoBuf_Include_Path>/foo/bar.proto" instead of
# "<ProtoBuf_Include_Path>/third_party/foo/bar.proto". This ensures
# correct symbol being generated and C++ include path being correct.
# More info can be found at https://github.com/grpc/grpc/pull/25272.
#
function(protobuf_generate_grpc_cpp_with_import_path_correction FILE_LOCATION IMPORT_PATH)
if(NOT FILE_LOCATION)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files")
return()
endif()
# Sets the include path for ProtoBuf files
set(_protobuf_include_path -I . -I ${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR})
# The absolute path of the expected place for the input proto file
# For example, health proto has package name grpc.health.v1, it's expected to be:
# ${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1/health.proto
get_filename_component(ABS_FIL ${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH} ABSOLUTE)
# Get the name of the file, which used to generate output file names for
# this command.
# Example: "health" for "health.proto"
get_filename_component(FIL_WE ${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH} NAME_WE)
# Get the relative path between the expected place for the proto and the
# working directory. In normal cases, it would be equal IMPORT_PATH, but
# it's better to be agnostic to all the global folder locations (like the
# centralized location ${_gRPC_PROTO_SRCS_DIR}).
# Example: grpc/health/v1/health.proto
file(RELATIVE_PATH REL_FIL ${_gRPC_PROTO_SRCS_DIR} ${ABS_FIL})
# Get the directory of the relative path.
# Example: grpc/health/v1
get_filename_component(REL_DIR ${REL_FIL} DIRECTORY)
# Get the directory and name for output filenames generation.
# Example: "grpc/health/v1/health", the file name extension is omitted.
set(RELFIL_WE "${REL_DIR}/${FIL_WE}")
# Copy the proto file to a centralized location, with the correct import
# path. For example, health proto has package name grpc.health.v1, the bash
# equivalent would be:
# cp src/proto/grpc/health/v1/health.proto ${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1
file(MAKE_DIRECTORY ${_gRPC_PROTO_SRCS_DIR}/${REL_DIR})
file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION} DESTINATION ${_gRPC_PROTO_SRCS_DIR}/${REL_DIR})
#if cross-compiling, find host plugin
if(CMAKE_CROSSCOMPILING)
find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin)
else()
set(_gRPC_CPP_PLUGIN $<TARGET_FILE:grpc_cpp_plugin>)
endif()
add_custom_command(
OUTPUT "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc"
"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h"
COMMAND ${_gRPC_PROTOBUF_PROTOC_EXECUTABLE}
ARGS --grpc_out=generate_mock_code=true:${_gRPC_PROTO_GENS_DIR}
--cpp_out=${_gRPC_PROTO_GENS_DIR}
--plugin=protoc-gen-grpc=${_gRPC_CPP_PLUGIN}
${_protobuf_include_path}
${REL_FIL}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION} ${ABS_FIL} ${_gRPC_PROTOBUF_PROTOC} grpc_cpp_plugin
WORKING_DIRECTORY ${_gRPC_PROTO_SRCS_DIR}
COMMENT "Running gRPC C++ protocol buffer compiler for ${IMPORT_PATH}"
VERBATIM)
endfunction()
# These options allow users to enable or disable the building of the various
# protoc plugins. For example, running CMake with
# -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF will disable building the C# plugin.
@ -427,185 +562,185 @@ add_custom_target(tools_cxx
add_custom_target(tools
DEPENDS tools_c tools_cxx)
protobuf_generate_grpc_cpp(
src/proto/grpc/channelz/channelz.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/channelz/channelz.proto src/proto/grpc/channelz/channelz.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/core/stats.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/core/stats.proto src/proto/grpc/core/stats.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/health/v1/health.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/health/v1/health.proto src/proto/grpc/health/v1/health.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/lb/v1/load_balancer.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/lb/v1/load_balancer.proto src/proto/grpc/lb/v1/load_balancer.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/lookup/v1/rls.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/lookup/v1/rls.proto src/proto/grpc/lookup/v1/rls.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/lookup/v1/rls_config.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/lookup/v1/rls_config.proto src/proto/grpc/lookup/v1/rls_config.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/reflection/v1alpha/reflection.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/reflection/v1alpha/reflection.proto src/proto/grpc/reflection/v1alpha/reflection.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/status/status.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/status/status.proto src/proto/grpc/status/status.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/benchmark_service.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/benchmark_service.proto src/proto/grpc/testing/benchmark_service.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/control.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/control.proto src/proto/grpc/testing/control.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/duplicate/echo_duplicate.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/duplicate/echo_duplicate.proto src/proto/grpc/testing/duplicate/echo_duplicate.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/echo.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/echo.proto src/proto/grpc/testing/echo.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/echo_messages.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/echo_messages.proto src/proto/grpc/testing/echo_messages.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/empty.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/empty.proto src/proto/grpc/testing/empty.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/messages.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/messages.proto src/proto/grpc/testing/messages.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/payloads.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/payloads.proto src/proto/grpc/testing/payloads.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/report_qps_scenario_service.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/report_qps_scenario_service.proto src/proto/grpc/testing/report_qps_scenario_service.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/simple_messages.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/simple_messages.proto src/proto/grpc/testing/simple_messages.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/stats.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/stats.proto src/proto/grpc/testing/stats.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/test.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/test.proto src/proto/grpc/testing/test.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/worker_service.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/worker_service.proto src/proto/grpc/testing/worker_service.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/ads_for_test.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/ads_for_test.proto src/proto/grpc/testing/xds/ads_for_test.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/cds_for_test.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/cds_for_test.proto src/proto/grpc/testing/xds/cds_for_test.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/eds_for_test.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/eds_for_test.proto src/proto/grpc/testing/xds/eds_for_test.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/lds_rds_for_test.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/lds_rds_for_test.proto src/proto/grpc/testing/xds/lds_rds_for_test.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/lrs_for_test.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/lrs_for_test.proto src/proto/grpc/testing/xds/lrs_for_test.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/address.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/address.proto src/proto/grpc/testing/xds/v3/address.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/ads.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/ads.proto src/proto/grpc/testing/xds/v3/ads.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/aggregate_cluster.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/aggregate_cluster.proto src/proto/grpc/testing/xds/v3/aggregate_cluster.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/base.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/base.proto src/proto/grpc/testing/xds/v3/base.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/cluster.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/cluster.proto src/proto/grpc/testing/xds/v3/cluster.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/config_dump.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/config_dump.proto src/proto/grpc/testing/xds/v3/config_dump.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/config_source.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/config_source.proto src/proto/grpc/testing/xds/v3/config_source.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/csds.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/csds.proto src/proto/grpc/testing/xds/v3/csds.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/discovery.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/discovery.proto src/proto/grpc/testing/xds/v3/discovery.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/endpoint.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/endpoint.proto src/proto/grpc/testing/xds/v3/endpoint.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/expr.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/expr.proto src/proto/grpc/testing/xds/v3/expr.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/extension.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/extension.proto src/proto/grpc/testing/xds/v3/extension.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/fault.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/fault.proto src/proto/grpc/testing/xds/v3/fault.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/fault_common.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/fault_common.proto src/proto/grpc/testing/xds/v3/fault_common.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/http_connection_manager.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/http_connection_manager.proto src/proto/grpc/testing/xds/v3/http_connection_manager.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/http_filter_rbac.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/http_filter_rbac.proto src/proto/grpc/testing/xds/v3/http_filter_rbac.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/listener.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/listener.proto src/proto/grpc/testing/xds/v3/listener.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/load_report.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/load_report.proto src/proto/grpc/testing/xds/v3/load_report.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/lrs.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/lrs.proto src/proto/grpc/testing/xds/v3/lrs.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/metadata.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/metadata.proto src/proto/grpc/testing/xds/v3/metadata.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/orca_load_report.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/orca_load_report.proto src/proto/grpc/testing/xds/v3/orca_load_report.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/orca_service.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/orca_service.proto src/proto/grpc/testing/xds/v3/orca_service.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/path.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/path.proto src/proto/grpc/testing/xds/v3/path.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/percent.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/percent.proto src/proto/grpc/testing/xds/v3/percent.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/protocol.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/protocol.proto src/proto/grpc/testing/xds/v3/protocol.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/range.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/range.proto src/proto/grpc/testing/xds/v3/range.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/rbac.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/rbac.proto src/proto/grpc/testing/xds/v3/rbac.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/regex.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/regex.proto src/proto/grpc/testing/xds/v3/regex.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/route.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/route.proto src/proto/grpc/testing/xds/v3/route.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/router.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/router.proto src/proto/grpc/testing/xds/v3/router.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/string.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/string.proto src/proto/grpc/testing/xds/v3/string.proto
)
protobuf_generate_grpc_cpp(
src/proto/grpc/testing/xds/v3/tls.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
src/proto/grpc/testing/xds/v3/tls.proto src/proto/grpc/testing/xds/v3/tls.proto
)
protobuf_generate_grpc_cpp(
test/core/tsi/alts/fake_handshaker/handshaker.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
test/core/tsi/alts/fake_handshaker/handshaker.proto test/core/tsi/alts/fake_handshaker/handshaker.proto
)
protobuf_generate_grpc_cpp(
test/core/tsi/alts/fake_handshaker/transport_security_common.proto
protobuf_generate_grpc_cpp_with_import_path_correction(
test/core/tsi/alts/fake_handshaker/transport_security_common.proto test/core/tsi/alts/fake_handshaker/transport_security_common.proto
)
if(gRPC_BUILD_TESTS)

@ -8572,4 +8572,33 @@ targets:
- grpcpp_channelz
- grpc_test_util
- grpc++_test_config
external_proto_libraries:
- destination: third_party/envoy-api
hash: c5807010b67033330915ca5a20483e30538ae5e689aa14b3631d6284beca4630
proto_prefix: third_party/envoy-api/
strip_prefix: data-plane-api-9c42588c956220b48eb3099d186487c2f04d32ec
urls:
- https://storage.googleapis.com/grpc-bazel-mirror/github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz
- https://github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz
- destination: third_party/googleapis
hash: 5bb6b0253ccf64b53d6c7249625a7e3f6c3bc6402abd52d3778bfa48258703a0
proto_prefix: third_party/googleapis/
strip_prefix: googleapis-2f9af297c84c55c8b871ba4495e01ade42476c92
urls:
- https://storage.googleapis.com/grpc-bazel-mirror/github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz
- https://github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz
- destination: third_party/opencensus-proto/src
hash: b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0
proto_prefix: third_party/opencensus-proto/src/
strip_prefix: opencensus-proto-0.3.0/src
urls:
- https://storage.googleapis.com/grpc-bazel-mirror/github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz
- https://github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz
- destination: third_party/xds
hash: 5bc8365613fe2f8ce6cc33959b7667b13b7fe56cb9d16ba740c06e1a7c4242fc
proto_prefix: third_party/xds/
strip_prefix: xds-cb28da3451f158a947dfc45090fe92b07b243bc1
urls:
- https://storage.googleapis.com/grpc-bazel-mirror/github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz
- https://github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz
tests: []

@ -0,0 +1,40 @@
# Copyright 2021 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set(_download_archive_TEMPORARY_DIR ${CMAKE_BINARY_DIR}/http_archives)
file(MAKE_DIRECTORY ${_download_archive_TEMPORARY_DIR})
# This is basically Bazel's http_archive.
# Note that strip_prefix strips the directory path prefix of the extracted
# archive content, and it may strip multiple directories.
function(download_archive destination url hash strip_prefix)
# Fetch and validate
set(_TEMPORARY_FILE ${_download_archive_TEMPORARY_DIR}/${strip_prefix}.tar.gz)
message(STATUS "Downloading from ${url}, if failed, please try configuring again")
file(DOWNLOAD ${url} ${_TEMPORARY_FILE}
TIMEOUT 60
EXPECTED_HASH SHA256=${hash}
TLS_VERIFY ON)
# Extract
execute_process(COMMAND
${CMAKE_COMMAND} -E tar xvf ${_TEMPORARY_FILE}
WORKING_DIRECTORY ${_download_archive_TEMPORARY_DIR}
OUTPUT_QUIET)
get_filename_component(_download_archive_Destination_Path ${destination} DIRECTORY)
file(MAKE_DIRECTORY ${_download_archive_Destination_Path})
file(RENAME ${_download_archive_TEMPORARY_DIR}/${strip_prefix} ${destination})
# Clean up
file(REMOVE ${_download_archive_TEMPORARY_DIR}/${strip_prefix})
file(REMOVE ${_TEMPORARY_FILE})
endfunction()

@ -27,14 +27,23 @@
proto_re = re.compile('(.*)\\.proto')
lib_map = {lib.name: lib for lib in libs}
third_party_proto_prefixes = {lib.proto_prefix for lib in external_proto_libraries}
def third_party_proto_import_path(path):
"""Removes third_party prefix to match ProtoBuf's relative import path."""
for prefix in third_party_proto_prefixes:
if path.startswith(prefix):
return path[len(prefix):]
return path
def proto_replace_ext(filename, ext):
m = proto_re.match(filename)
if not m:
return filename
return '${_gRPC_PROTO_GENS_DIR}/' + m.group(1) + ext
return '${_gRPC_PROTO_GENS_DIR}/' + third_party_proto_import_path(m.group(1)) + ext
def is_absl_lib(lib_name):
return lib_name.startswith("absl/");
return lib_name.startswith("absl/")
def get_absl_dep(lib_name):
return lib_map[lib_name].cmake_target
@ -356,11 +365,34 @@
include(cmake/upb.cmake)
include(cmake/xxhash.cmake)
include(cmake/zlib.cmake)
include(cmake/download_archive.cmake)
% for external_proto_library in external_proto_libraries:
% if len(external_proto_library.urls) > 1:
# Setup external proto library at ${external_proto_library.destination} with ${len(external_proto_library.urls)} download URLs
% else:
# Setup external proto library at ${external_proto_library.destination} if it doesn't exist
% endif
% for download_url in external_proto_library.urls:
if (NOT EXISTS <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>/${external_proto_library.destination})
# Download the archive via HTTP, validate the checksum, and extract to ${external_proto_library.destination}.
download_archive(
<%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>/${external_proto_library.destination}
${download_url}
${external_proto_library.hash}
${external_proto_library.strip_prefix}
)
endif()
% endfor
% endfor
if(WIN32)
set(_gRPC_BASELIB_LIBRARIES ws2_32 crypt32)
endif()
# Create directory for proto source files
set(_gRPC_PROTO_SRCS_DIR <%text>${CMAKE_BINARY_DIR}/protos</%text>)
file(MAKE_DIRECTORY <%text>${_gRPC_PROTO_SRCS_DIR}</%text>)
# Create directory for generated .proto files
set(_gRPC_PROTO_GENS_DIR <%text>${CMAKE_BINARY_DIR}/gens</%text>)
file(MAKE_DIRECTORY <%text>${_gRPC_PROTO_GENS_DIR}</%text>)
@ -368,6 +400,10 @@
# protobuf_generate_grpc_cpp
# --------------------------
#
# This method is no longer used by gRPC's CMake build process. However, it
# is used by many open source dependencies, that we might want to keep
# backward compatibility here.
#
# Add custom commands to process ``.proto`` files to C++ using protoc and
# GRPC plugin::
#
@ -382,40 +418,90 @@
return()
endif()
set(_protobuf_include_path -I . -I <%text>${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}</%text>)
foreach(FIL <%text>${ARGN}</%text>)
get_filename_component(ABS_FIL <%text>${FIL}</%text> ABSOLUTE)
get_filename_component(FIL_WE <%text>${FIL}</%text> NAME_WE)
file(RELATIVE_PATH REL_FIL <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text> <%text>${ABS_FIL}</%text>)
get_filename_component(REL_DIR <%text>${REL_FIL}</%text> DIRECTORY)
set(RELFIL_WE "<%text>${REL_DIR}/${FIL_WE}</%text>")
#if cross-compiling, find host plugin
if(CMAKE_CROSSCOMPILING)
find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin)
else()
set(_gRPC_CPP_PLUGIN $<TARGET_FILE:grpc_cpp_plugin>)
endif()
add_custom_command(
OUTPUT <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h"</%text>
COMMAND <%text>${_gRPC_PROTOBUF_PROTOC_EXECUTABLE}</%text>
ARGS --grpc_out=<%text>generate_mock_code=true:${_gRPC_PROTO_GENS_DIR}</%text>
--cpp_out=<%text>${_gRPC_PROTO_GENS_DIR}</%text>
--plugin=protoc-gen-grpc=<%text>${_gRPC_CPP_PLUGIN}</%text>
<%text>${_protobuf_include_path}</%text>
<%text>${REL_FIL}</%text>
DEPENDS <%text>${ABS_FIL}</%text> <%text>${_gRPC_PROTOBUF_PROTOC}</%text> <%text>${_gRPC_CPP_PLUGIN}</%text>
WORKING_DIRECTORY <%text>${CMAKE_CURRENT_SOURCE_DIR}</%text>
COMMENT "Running gRPC C++ protocol buffer compiler on <%text>${FIL}</%text>"
VERBATIM)
protobuf_generate_grpc_cpp_with_import_path_correction(<%text>${FIL}</%text> <%text>${FIL}</%text>)
endforeach()
endfunction()
# protobuf_generate_grpc_cpp_with_import_path_correction
# --------------------------
#
# Add custom commands to process ``.proto`` files to C++ using protoc and
# GRPC plugin::
#
# protobuf_generate_grpc_cpp_with_import_path_correction <FILE_LOCATION> <IMPORT_PATH>
#
# ``FILE_LOCATION``
# The relative path of the ``.proto`` file to the project root
# ``IMPORT_PATH``
# The proto import path that itself expected to be placed in. For
# example, a "bar.proto" file wants to be imported as
# `import "foo/bar.proto"`. Then we should place it under
# "<ProtoBuf_Include_Path>/foo/bar.proto" instead of
# "<ProtoBuf_Include_Path>/third_party/foo/bar.proto". This ensures
# correct symbol being generated and C++ include path being correct.
# More info can be found at https://github.com/grpc/grpc/pull/25272.
#
function(protobuf_generate_grpc_cpp_with_import_path_correction FILE_LOCATION IMPORT_PATH)
if(NOT FILE_LOCATION)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files")
return()
endif()
# Sets the include path for ProtoBuf files
set(_protobuf_include_path -I . -I <%text>${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}</%text>)
# The absolute path of the expected place for the input proto file
# For example, health proto has package name grpc.health.v1, it's expected to be:
# <%text>${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1/health.proto</%text>
get_filename_component(ABS_FIL <%text>${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH}</%text> ABSOLUTE)
# Get the name of the file, which used to generate output file names for
# this command.
# Example: "health" for "health.proto"
get_filename_component(FIL_WE <%text>${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH}</%text> NAME_WE)
# Get the relative path between the expected place for the proto and the
# working directory. In normal cases, it would be equal IMPORT_PATH, but
# it's better to be agnostic to all the global folder locations (like the
# centralized location <%text>${_gRPC_PROTO_SRCS_DIR})</%text>.
# Example: grpc/health/v1/health.proto
file(RELATIVE_PATH REL_FIL <%text>${_gRPC_PROTO_SRCS_DIR}</%text> <%text>${ABS_FIL}</%text>)
# Get the directory of the relative path.
# Example: grpc/health/v1
get_filename_component(REL_DIR <%text>${REL_FIL}</%text> DIRECTORY)
# Get the directory and name for output filenames generation.
# Example: "grpc/health/v1/health", the file name extension is omitted.
set(RELFIL_WE "<%text>${REL_DIR}/${FIL_WE}</%text>")
# Copy the proto file to a centralized location, with the correct import
# path. For example, health proto has package name grpc.health.v1, the bash
# equivalent would be:
# cp src/proto/grpc/health/v1/health.proto <%text>${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1</%text>
file(MAKE_DIRECTORY <%text>${_gRPC_PROTO_SRCS_DIR}/${REL_DIR}</%text>)
file(COPY <%text>${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION}</%text> DESTINATION <%text>${_gRPC_PROTO_SRCS_DIR}/${REL_DIR}</%text>)
#if cross-compiling, find host plugin
if(CMAKE_CROSSCOMPILING)
find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin)
else()
set(_gRPC_CPP_PLUGIN $<TARGET_FILE:grpc_cpp_plugin>)
endif()
add_custom_command(
OUTPUT <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc"</%text>
<%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h"</%text>
COMMAND <%text>${_gRPC_PROTOBUF_PROTOC_EXECUTABLE}</%text>
ARGS --grpc_out=<%text>generate_mock_code=true:${_gRPC_PROTO_GENS_DIR}</%text>
--cpp_out=<%text>${_gRPC_PROTO_GENS_DIR}</%text>
--plugin=protoc-gen-grpc=<%text>${_gRPC_CPP_PLUGIN}</%text>
<%text>${_protobuf_include_path}</%text>
<%text>${REL_FIL}</%text>
DEPENDS <%text>${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION}</%text> <%text>${ABS_FIL}</%text> <%text>${_gRPC_PROTOBUF_PROTOC}</%text> grpc_cpp_plugin
WORKING_DIRECTORY <%text>${_gRPC_PROTO_SRCS_DIR}</%text>
COMMENT "Running gRPC C++ protocol buffer compiler for <%text>${IMPORT_PATH}</%text>"
VERBATIM)
endfunction()
# These options allow users to enable or disable the building of the various
# protoc plugins. For example, running CMake with
# -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF will disable building the C# plugin.
@ -455,8 +541,8 @@
DEPENDS tools_c tools_cxx)
% for src in sorted(protobuf_gen_files):
protobuf_generate_grpc_cpp(
${src}
protobuf_generate_grpc_cpp_with_import_path_correction(
${src} ${third_party_proto_import_path(src)}
)
% endfor

@ -68,7 +68,11 @@ popd
@rem Just before installing gRPC, wipe out contents of all the submodules to simulate
@rem a standalone build from an archive
git submodule deinit --all --force
@rem NOTE(lidiz) We used to use "git submodule deinit", but it leaves an empty
@rem folder for deinit-ed submodules, blocking the CMake download. For users
@rem downloaded gRPC code as an archive, they won't have submodule residual
@rem folders, like the following command trying to imitate.
git submodule foreach bash -c "cd $toplevel; rm -rf $name"
@rem Install gRPC
mkdir cmake\build

@ -30,11 +30,11 @@
# format entirely or simplify it to a point where it becomes self-explanatory
# and doesn't need any detailed documentation.
import collections
from dataclasses import asdict
from dataclasses import dataclass
from dataclasses import field
import os
import re
import subprocess
import sys
from typing import Any, Dict, Iterable, List, Optional
import xml.etree.ElementTree as ET
@ -45,6 +45,48 @@ BuildMetadata = Dict[str, Any]
BuildDict = Dict[str, BuildMetadata]
BuildYaml = Dict[str, Any]
BuildMetadata = Dict[str, Any]
BuildDict = Dict[str, BuildMetadata]
BuildYaml = Dict[str, Any]
# This is basically a Python dict with predefined fields and types
@dataclass()
class ExternalProtoLibrary:
# The relative path of this proto library should be. Preferably, it should
# match the submodule path.
destination: str
# The prefix to remove in order to insure the proto import is correct. For
# more info, see description of https://github.com/grpc/grpc/pull/25272.
proto_prefix: str
# Following 3 fields should be filled by build metadata from Bazel.
urls: List[str] = field(default_factory=list)
hash: str = ''
strip_prefix: str = ''
EXTERNAL_PROTO_LIBRARIES = {
'envoy_api':
ExternalProtoLibrary(destination='third_party/envoy-api',
proto_prefix='third_party/envoy-api/'),
'com_google_googleapis':
ExternalProtoLibrary(destination='third_party/googleapis',
proto_prefix='third_party/googleapis/'),
'com_github_cncf_udpa':
ExternalProtoLibrary(destination='third_party/xds',
proto_prefix='third_party/xds/'),
'opencensus_proto':
ExternalProtoLibrary(destination='third_party/opencensus-proto/src',
proto_prefix='third_party/opencensus-proto/src/'),
}
def _maybe_get_internal_path(name: str) -> Optional[str]:
for key in EXTERNAL_PROTO_LIBRARIES:
if name.startswith('@' + key):
return key
return None
def _bazel_query_xml_tree(query: str) -> ET.Element:
"""Get xml output of bazel query invocation, parsed as XML tree"""
@ -98,6 +140,7 @@ def _extract_rules_from_bazel_xml(xml_tree):
'cc_binary',
'cc_test',
'cc_proto_library',
'cc_proto_gen_validate',
'proto_library',
'upb_proto_library',
'upb_proto_reflection_library',
@ -109,6 +152,8 @@ def _extract_rules_from_bazel_xml(xml_tree):
def _get_bazel_label(target_name: str) -> str:
if target_name.startswith('@'):
return target_name
if ':' in target_name:
return '//%s' % target_name
else:
@ -149,17 +194,40 @@ def _extract_nonpublic_headers(bazel_rule: BuildMetadata) -> List[str]:
def _extract_sources(bazel_rule: BuildMetadata) -> List[str]:
"""Gets list of source files from a bazel rule"""
result = []
for dep in bazel_rule['srcs']:
if dep.startswith('//') and (dep.endswith('.cc') or dep.endswith('.c')
or dep.endswith('.proto')):
result.append(_extract_source_file_path(dep))
for src in bazel_rule['srcs']:
if src.endswith('.cc') or src.endswith('.c') or src.endswith('.proto'):
if src.startswith('//'):
# This source file is local to gRPC
result.append(_extract_source_file_path(src))
else:
# This source file is external, and we need to translate the
# @REPO_NAME to a valid path prefix. At this stage, we need
# to check repo name, since the label/path mapping is not
# available in BUILD files.
external_proto_library_name = _maybe_get_internal_path(src)
if external_proto_library_name is not None:
result.append(
src.replace(
f'@{external_proto_library_name}//',
EXTERNAL_PROTO_LIBRARIES[
external_proto_library_name].proto_prefix).
replace(':', '/'))
return list(sorted(result))
def _extract_deps(bazel_rule: BuildMetadata,
bazel_rules: BuildDict) -> List[str]:
"""Gets list of deps from from a bazel rule"""
return list(sorted(bazel_rule['deps']))
deps = set(bazel_rule['deps'])
for src in bazel_rule['srcs']:
if not src.endswith('.cc') and not src.endswith(
'.c') and not src.endswith('.proto'):
if src in bazel_rules:
# This label doesn't point to a source file, but another Bazel
# target. This is required for :pkg_cc_proto_validate targets,
# and it's generally allowed by Bazel.
deps.add(src)
return list(sorted(list(deps)))
def _create_target_from_bazel_rule(target_name: str,
@ -262,7 +330,6 @@ def _compute_transitive_metadata(
# This item is not processed before, compute now
_compute_transitive_metadata(dep, bazel_rules,
bazel_label_to_dep_name)
transitive_deps.update(bazel_rules[dep].get(
'_TRANSITIVE_DEPS', []))
collapsed_deps.update(
@ -336,15 +403,16 @@ def _compute_transitive_metadata(
bazel_rule['_EXCLUDE_DEPS'] = list(sorted(exclude_deps))
# TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a slightly different logic)
# TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a
# slightly different logic)
# TODO(jtattermusch): This is done to avoid introducing too many intermediate
# libraries into the build.yaml-based builds (which might in cause issues
# building language-specific artifacts) and also because the libraries
# in build.yaml-based build are generally considered units of distributions
# (= public libraries that are visible to the user and are installable),
# while in bazel builds it is customary to define larger number of smaller
# "sublibraries". The need for elision (and expansion)
# of intermediate libraries can be re-evaluated in the future.
# building language-specific artifacts) and also because the libraries in
# build.yaml-based build are generally considered units of distributions (=
# public libraries that are visible to the user and are installable), while in
# bazel builds it is customary to define larger number of smaller
# "sublibraries". The need for elision (and expansion) of intermediate libraries
# can be re-evaluated in the future.
def _populate_transitive_metadata(bazel_rules: Any,
public_dep_names: Iterable[str]) -> None:
"""Add 'transitive_deps' field for each of the rules"""
@ -722,6 +790,48 @@ def _generate_build_extra_metadata_for_tests(
return test_metadata
def _parse_http_archives(xml_tree: ET.Element) -> List[ExternalProtoLibrary]:
"""Parse Bazel http_archive rule into ExternalProtoLibrary objects."""
result = []
for xml_http_archive in xml_tree:
if xml_http_archive.tag != 'rule' or xml_http_archive.attrib[
'class'] != 'http_archive':
continue
# A distilled Python representation of Bazel http_archive
http_archive = dict()
for xml_node in xml_http_archive:
if xml_node.attrib['name'] == 'name':
http_archive["name"] = xml_node.attrib['value']
if xml_node.attrib['name'] == 'urls':
http_archive["urls"] = []
for url_node in xml_node:
http_archive["urls"].append(url_node.attrib['value'])
if xml_node.attrib['name'] == 'url':
http_archive["urls"] = [xml_node.attrib['value']]
if xml_node.attrib['name'] == 'sha256':
http_archive["hash"] = xml_node.attrib['value']
if xml_node.attrib['name'] == 'strip_prefix':
http_archive["strip_prefix"] = xml_node.attrib['value']
if http_archive["name"] not in EXTERNAL_PROTO_LIBRARIES:
# If this http archive is not one of the external proto libraries,
# we don't want to include it as a CMake target
continue
lib = EXTERNAL_PROTO_LIBRARIES[http_archive["name"]]
lib.urls = http_archive["urls"]
lib.hash = http_archive["hash"]
lib.strip_prefix = http_archive["strip_prefix"]
result.append(lib)
return result
def _generate_external_proto_libraries() -> List[Dict[str, Any]]:
"""Generates the build metadata for external proto libraries"""
xml_tree = _bazel_query_xml_tree('kind(http_archive, //external:*)')
libraries = _parse_http_archives(xml_tree)
libraries.sort(key=lambda x: x.destination)
return list(map(asdict, libraries))
def _detect_and_print_issues(build_yaml_like: BuildYaml) -> None:
"""Try detecting some unusual situations and warn about them."""
for tgt in build_yaml_like['targets']:
@ -1103,6 +1213,15 @@ all_targets_dict = _generate_build_metadata(all_extra_metadata, bazel_rules)
# 'tests': { TARGET_DICT_FOR_TEST_XYZ, ...} }
build_yaml_like = _convert_to_build_yaml_like(all_targets_dict)
# Step 7: generates build metadata for external ProtoBuf libraries.
# We only want the ProtoBuf sources from these ProtoBuf dependencies, which may
# not be present in our release source tar balls. These rules will be used in CMake
# to download these libraries if not existed. Even if the download failed, it
# will be a soft error that doesn't block existing target from successfully
# built.
build_yaml_like[
'external_proto_libraries'] = _generate_external_proto_libraries()
# detect and report some suspicious situations we've seen before
_detect_and_print_issues(build_yaml_like)

@ -89,6 +89,13 @@ VALID_ATTRIBUTE_KEYS_MAP = {
'vs_proj_dir': anything(),
'zlib': one_of((True,)),
},
'external_proto_library': {
'destination': anything(),
'proto_prefix': anything(),
'urls': anything(),
'hash': anything(),
'strip_prefix': anything(),
}
}
@ -113,10 +120,10 @@ def check_attributes(entity, kind, errors):
def mako_plugin(dictionary):
"""The exported plugin code for check_attr.
This validates that filegroups, libs, and target can have only valid
attributes. This is mainly for preventing build.yaml from having
unnecessary and misleading attributes accidentally.
"""
This validates that filegroups, libs, and target can have only valid
attributes. This is mainly for preventing build.yaml from having
unnecessary and misleading attributes accidentally.
"""
errors = []
for filegroup in dictionary.get('filegroups', {}):
@ -125,5 +132,7 @@ def mako_plugin(dictionary):
check_attributes(lib, 'lib', errors)
for target in dictionary.get('targets', {}):
check_attributes(target, 'target', errors)
for target in dictionary.get('external_proto_libraries', {}):
check_attributes(target, 'external_proto_library', errors)
if errors:
raise Exception('\n'.join(errors))

Loading…
Cancel
Save