diff --git a/CMakeLists.txt b/CMakeLists.txt index bbe64bd4295..a028a7aa91e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -318,11 +318,92 @@ include(cmake/ssl.cmake) include(cmake/upb.cmake) include(cmake/xxhash.cmake) include(cmake/zlib.cmake) +include(cmake/download_archive.cmake) + +# Setup external proto library at third_party/envoy-api with 2 download URLs +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api) + # Download the archive via HTTP, validate the checksum, and extract to third_party/envoy-api. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api + https://storage.googleapis.com/grpc-bazel-mirror/github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz + c5807010b67033330915ca5a20483e30538ae5e689aa14b3631d6284beca4630 + data-plane-api-9c42588c956220b48eb3099d186487c2f04d32ec + ) +endif() +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api) + # Download the archive via HTTP, validate the checksum, and extract to third_party/envoy-api. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/envoy-api + https://github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz + c5807010b67033330915ca5a20483e30538ae5e689aa14b3631d6284beca4630 + data-plane-api-9c42588c956220b48eb3099d186487c2f04d32ec + ) +endif() +# Setup external proto library at third_party/googleapis with 2 download URLs +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis) + # Download the archive via HTTP, validate the checksum, and extract to third_party/googleapis. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis + https://storage.googleapis.com/grpc-bazel-mirror/github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz + 5bb6b0253ccf64b53d6c7249625a7e3f6c3bc6402abd52d3778bfa48258703a0 + googleapis-2f9af297c84c55c8b871ba4495e01ade42476c92 + ) +endif() +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis) + # Download the archive via HTTP, validate the checksum, and extract to third_party/googleapis. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/googleapis + https://github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz + 5bb6b0253ccf64b53d6c7249625a7e3f6c3bc6402abd52d3778bfa48258703a0 + googleapis-2f9af297c84c55c8b871ba4495e01ade42476c92 + ) +endif() +# Setup external proto library at third_party/opencensus-proto/src with 2 download URLs +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src) + # Download the archive via HTTP, validate the checksum, and extract to third_party/opencensus-proto/src. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src + https://storage.googleapis.com/grpc-bazel-mirror/github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz + b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0 + opencensus-proto-0.3.0/src + ) +endif() +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src) + # Download the archive via HTTP, validate the checksum, and extract to third_party/opencensus-proto/src. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/opencensus-proto/src + https://github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz + b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0 + opencensus-proto-0.3.0/src + ) +endif() +# Setup external proto library at third_party/xds with 2 download URLs +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds) + # Download the archive via HTTP, validate the checksum, and extract to third_party/xds. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds + https://storage.googleapis.com/grpc-bazel-mirror/github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz + 5bc8365613fe2f8ce6cc33959b7667b13b7fe56cb9d16ba740c06e1a7c4242fc + xds-cb28da3451f158a947dfc45090fe92b07b243bc1 + ) +endif() +if (NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds) + # Download the archive via HTTP, validate the checksum, and extract to third_party/xds. + download_archive( + ${CMAKE_CURRENT_SOURCE_DIR}/third_party/xds + https://github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz + 5bc8365613fe2f8ce6cc33959b7667b13b7fe56cb9d16ba740c06e1a7c4242fc + xds-cb28da3451f158a947dfc45090fe92b07b243bc1 + ) +endif() if(WIN32) set(_gRPC_BASELIB_LIBRARIES ws2_32 crypt32) endif() +# Create directory for proto source files +set(_gRPC_PROTO_SRCS_DIR ${CMAKE_BINARY_DIR}/protos) +file(MAKE_DIRECTORY ${_gRPC_PROTO_SRCS_DIR}) # Create directory for generated .proto files set(_gRPC_PROTO_GENS_DIR ${CMAKE_BINARY_DIR}/gens) file(MAKE_DIRECTORY ${_gRPC_PROTO_GENS_DIR}) @@ -330,6 +411,10 @@ file(MAKE_DIRECTORY ${_gRPC_PROTO_GENS_DIR}) # protobuf_generate_grpc_cpp # -------------------------- # +# This method is no longer used by gRPC's CMake build process. However, it +# is used by many open source dependencies, that we might want to keep +# backward compatibility here. +# # Add custom commands to process ``.proto`` files to C++ using protoc and # GRPC plugin:: # @@ -344,40 +429,90 @@ function(protobuf_generate_grpc_cpp) return() endif() - set(_protobuf_include_path -I . -I ${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}) foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(FIL_WE ${FIL} NAME_WE) - file(RELATIVE_PATH REL_FIL ${CMAKE_CURRENT_SOURCE_DIR} ${ABS_FIL}) - get_filename_component(REL_DIR ${REL_FIL} DIRECTORY) - set(RELFIL_WE "${REL_DIR}/${FIL_WE}") - - #if cross-compiling, find host plugin - if(CMAKE_CROSSCOMPILING) - find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin) - else() - set(_gRPC_CPP_PLUGIN $) - endif() - - add_custom_command( - OUTPUT "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" - "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" - "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" - "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" - "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" - COMMAND ${_gRPC_PROTOBUF_PROTOC_EXECUTABLE} - ARGS --grpc_out=generate_mock_code=true:${_gRPC_PROTO_GENS_DIR} - --cpp_out=${_gRPC_PROTO_GENS_DIR} - --plugin=protoc-gen-grpc=${_gRPC_CPP_PLUGIN} - ${_protobuf_include_path} - ${REL_FIL} - DEPENDS ${ABS_FIL} ${_gRPC_PROTOBUF_PROTOC} ${_gRPC_CPP_PLUGIN} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}" - VERBATIM) + protobuf_generate_grpc_cpp_with_import_path_correction(${FIL} ${FIL}) endforeach() endfunction() +# protobuf_generate_grpc_cpp_with_import_path_correction +# -------------------------- +# +# Add custom commands to process ``.proto`` files to C++ using protoc and +# GRPC plugin:: +# +# protobuf_generate_grpc_cpp_with_import_path_correction +# +# ``FILE_LOCATION`` +# The relative path of the ``.proto`` file to the project root +# ``IMPORT_PATH`` +# The proto import path that itself expected to be placed in. For +# example, a "bar.proto" file wants to be imported as +# `import "foo/bar.proto"`. Then we should place it under +# "/foo/bar.proto" instead of +# "/third_party/foo/bar.proto". This ensures +# correct symbol being generated and C++ include path being correct. +# More info can be found at https://github.com/grpc/grpc/pull/25272. +# +function(protobuf_generate_grpc_cpp_with_import_path_correction FILE_LOCATION IMPORT_PATH) + if(NOT FILE_LOCATION) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") + return() + endif() + + # Sets the include path for ProtoBuf files + set(_protobuf_include_path -I . -I ${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}) + # The absolute path of the expected place for the input proto file + # For example, health proto has package name grpc.health.v1, it's expected to be: + # ${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1/health.proto + get_filename_component(ABS_FIL ${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH} ABSOLUTE) + # Get the name of the file, which used to generate output file names for + # this command. + # Example: "health" for "health.proto" + get_filename_component(FIL_WE ${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH} NAME_WE) + # Get the relative path between the expected place for the proto and the + # working directory. In normal cases, it would be equal IMPORT_PATH, but + # it's better to be agnostic to all the global folder locations (like the + # centralized location ${_gRPC_PROTO_SRCS_DIR}). + # Example: grpc/health/v1/health.proto + file(RELATIVE_PATH REL_FIL ${_gRPC_PROTO_SRCS_DIR} ${ABS_FIL}) + # Get the directory of the relative path. + # Example: grpc/health/v1 + get_filename_component(REL_DIR ${REL_FIL} DIRECTORY) + # Get the directory and name for output filenames generation. + # Example: "grpc/health/v1/health", the file name extension is omitted. + set(RELFIL_WE "${REL_DIR}/${FIL_WE}") + # Copy the proto file to a centralized location, with the correct import + # path. For example, health proto has package name grpc.health.v1, the bash + # equivalent would be: + # cp src/proto/grpc/health/v1/health.proto ${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1 + file(MAKE_DIRECTORY ${_gRPC_PROTO_SRCS_DIR}/${REL_DIR}) + file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION} DESTINATION ${_gRPC_PROTO_SRCS_DIR}/${REL_DIR}) + + #if cross-compiling, find host plugin + if(CMAKE_CROSSCOMPILING) + find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin) + else() + set(_gRPC_CPP_PLUGIN $) + endif() + + add_custom_command( + OUTPUT "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" + "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" + "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" + "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" + "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" + COMMAND ${_gRPC_PROTOBUF_PROTOC_EXECUTABLE} + ARGS --grpc_out=generate_mock_code=true:${_gRPC_PROTO_GENS_DIR} + --cpp_out=${_gRPC_PROTO_GENS_DIR} + --plugin=protoc-gen-grpc=${_gRPC_CPP_PLUGIN} + ${_protobuf_include_path} + ${REL_FIL} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION} ${ABS_FIL} ${_gRPC_PROTOBUF_PROTOC} grpc_cpp_plugin + WORKING_DIRECTORY ${_gRPC_PROTO_SRCS_DIR} + COMMENT "Running gRPC C++ protocol buffer compiler for ${IMPORT_PATH}" + VERBATIM) +endfunction() + # These options allow users to enable or disable the building of the various # protoc plugins. For example, running CMake with # -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF will disable building the C# plugin. @@ -427,185 +562,185 @@ add_custom_target(tools_cxx add_custom_target(tools DEPENDS tools_c tools_cxx) -protobuf_generate_grpc_cpp( - src/proto/grpc/channelz/channelz.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/channelz/channelz.proto src/proto/grpc/channelz/channelz.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/core/stats.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/core/stats.proto src/proto/grpc/core/stats.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/health/v1/health.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/health/v1/health.proto src/proto/grpc/health/v1/health.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/lb/v1/load_balancer.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/lb/v1/load_balancer.proto src/proto/grpc/lb/v1/load_balancer.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/lookup/v1/rls.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/lookup/v1/rls.proto src/proto/grpc/lookup/v1/rls.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/lookup/v1/rls_config.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/lookup/v1/rls_config.proto src/proto/grpc/lookup/v1/rls_config.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/reflection/v1alpha/reflection.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/reflection/v1alpha/reflection.proto src/proto/grpc/reflection/v1alpha/reflection.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/status/status.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/status/status.proto src/proto/grpc/status/status.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/benchmark_service.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/benchmark_service.proto src/proto/grpc/testing/benchmark_service.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/control.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/control.proto src/proto/grpc/testing/control.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/duplicate/echo_duplicate.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/duplicate/echo_duplicate.proto src/proto/grpc/testing/duplicate/echo_duplicate.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/echo.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/echo.proto src/proto/grpc/testing/echo.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/echo_messages.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/echo_messages.proto src/proto/grpc/testing/echo_messages.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/empty.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/empty.proto src/proto/grpc/testing/empty.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/messages.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/messages.proto src/proto/grpc/testing/messages.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/payloads.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/payloads.proto src/proto/grpc/testing/payloads.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/report_qps_scenario_service.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/report_qps_scenario_service.proto src/proto/grpc/testing/report_qps_scenario_service.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/simple_messages.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/simple_messages.proto src/proto/grpc/testing/simple_messages.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/stats.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/stats.proto src/proto/grpc/testing/stats.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/test.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/test.proto src/proto/grpc/testing/test.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/worker_service.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/worker_service.proto src/proto/grpc/testing/worker_service.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/ads_for_test.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/ads_for_test.proto src/proto/grpc/testing/xds/ads_for_test.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/cds_for_test.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/cds_for_test.proto src/proto/grpc/testing/xds/cds_for_test.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/eds_for_test.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/eds_for_test.proto src/proto/grpc/testing/xds/eds_for_test.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/lds_rds_for_test.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/lds_rds_for_test.proto src/proto/grpc/testing/xds/lds_rds_for_test.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/lrs_for_test.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/lrs_for_test.proto src/proto/grpc/testing/xds/lrs_for_test.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/address.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/address.proto src/proto/grpc/testing/xds/v3/address.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/ads.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/ads.proto src/proto/grpc/testing/xds/v3/ads.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/aggregate_cluster.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/aggregate_cluster.proto src/proto/grpc/testing/xds/v3/aggregate_cluster.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/base.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/base.proto src/proto/grpc/testing/xds/v3/base.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/cluster.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/cluster.proto src/proto/grpc/testing/xds/v3/cluster.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/config_dump.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/config_dump.proto src/proto/grpc/testing/xds/v3/config_dump.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/config_source.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/config_source.proto src/proto/grpc/testing/xds/v3/config_source.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/csds.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/csds.proto src/proto/grpc/testing/xds/v3/csds.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/discovery.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/discovery.proto src/proto/grpc/testing/xds/v3/discovery.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/endpoint.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/endpoint.proto src/proto/grpc/testing/xds/v3/endpoint.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/expr.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/expr.proto src/proto/grpc/testing/xds/v3/expr.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/extension.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/extension.proto src/proto/grpc/testing/xds/v3/extension.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/fault.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/fault.proto src/proto/grpc/testing/xds/v3/fault.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/fault_common.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/fault_common.proto src/proto/grpc/testing/xds/v3/fault_common.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/http_connection_manager.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/http_connection_manager.proto src/proto/grpc/testing/xds/v3/http_connection_manager.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/http_filter_rbac.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/http_filter_rbac.proto src/proto/grpc/testing/xds/v3/http_filter_rbac.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/listener.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/listener.proto src/proto/grpc/testing/xds/v3/listener.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/load_report.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/load_report.proto src/proto/grpc/testing/xds/v3/load_report.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/lrs.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/lrs.proto src/proto/grpc/testing/xds/v3/lrs.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/metadata.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/metadata.proto src/proto/grpc/testing/xds/v3/metadata.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/orca_load_report.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/orca_load_report.proto src/proto/grpc/testing/xds/v3/orca_load_report.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/orca_service.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/orca_service.proto src/proto/grpc/testing/xds/v3/orca_service.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/path.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/path.proto src/proto/grpc/testing/xds/v3/path.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/percent.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/percent.proto src/proto/grpc/testing/xds/v3/percent.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/protocol.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/protocol.proto src/proto/grpc/testing/xds/v3/protocol.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/range.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/range.proto src/proto/grpc/testing/xds/v3/range.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/rbac.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/rbac.proto src/proto/grpc/testing/xds/v3/rbac.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/regex.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/regex.proto src/proto/grpc/testing/xds/v3/regex.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/route.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/route.proto src/proto/grpc/testing/xds/v3/route.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/router.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/router.proto src/proto/grpc/testing/xds/v3/router.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/string.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/string.proto src/proto/grpc/testing/xds/v3/string.proto ) -protobuf_generate_grpc_cpp( - src/proto/grpc/testing/xds/v3/tls.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + src/proto/grpc/testing/xds/v3/tls.proto src/proto/grpc/testing/xds/v3/tls.proto ) -protobuf_generate_grpc_cpp( - test/core/tsi/alts/fake_handshaker/handshaker.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + test/core/tsi/alts/fake_handshaker/handshaker.proto test/core/tsi/alts/fake_handshaker/handshaker.proto ) -protobuf_generate_grpc_cpp( - test/core/tsi/alts/fake_handshaker/transport_security_common.proto +protobuf_generate_grpc_cpp_with_import_path_correction( + test/core/tsi/alts/fake_handshaker/transport_security_common.proto test/core/tsi/alts/fake_handshaker/transport_security_common.proto ) if(gRPC_BUILD_TESTS) diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 8ec74b1dda3..77a8dcfe152 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -8572,4 +8572,33 @@ targets: - grpcpp_channelz - grpc_test_util - grpc++_test_config +external_proto_libraries: +- destination: third_party/envoy-api + hash: c5807010b67033330915ca5a20483e30538ae5e689aa14b3631d6284beca4630 + proto_prefix: third_party/envoy-api/ + strip_prefix: data-plane-api-9c42588c956220b48eb3099d186487c2f04d32ec + urls: + - https://storage.googleapis.com/grpc-bazel-mirror/github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz + - https://github.com/envoyproxy/data-plane-api/archive/9c42588c956220b48eb3099d186487c2f04d32ec.tar.gz +- destination: third_party/googleapis + hash: 5bb6b0253ccf64b53d6c7249625a7e3f6c3bc6402abd52d3778bfa48258703a0 + proto_prefix: third_party/googleapis/ + strip_prefix: googleapis-2f9af297c84c55c8b871ba4495e01ade42476c92 + urls: + - https://storage.googleapis.com/grpc-bazel-mirror/github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz + - https://github.com/googleapis/googleapis/archive/2f9af297c84c55c8b871ba4495e01ade42476c92.tar.gz +- destination: third_party/opencensus-proto/src + hash: b7e13f0b4259e80c3070b583c2f39e53153085a6918718b1c710caf7037572b0 + proto_prefix: third_party/opencensus-proto/src/ + strip_prefix: opencensus-proto-0.3.0/src + urls: + - https://storage.googleapis.com/grpc-bazel-mirror/github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz + - https://github.com/census-instrumentation/opencensus-proto/archive/v0.3.0.tar.gz +- destination: third_party/xds + hash: 5bc8365613fe2f8ce6cc33959b7667b13b7fe56cb9d16ba740c06e1a7c4242fc + proto_prefix: third_party/xds/ + strip_prefix: xds-cb28da3451f158a947dfc45090fe92b07b243bc1 + urls: + - https://storage.googleapis.com/grpc-bazel-mirror/github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz + - https://github.com/cncf/xds/archive/cb28da3451f158a947dfc45090fe92b07b243bc1.tar.gz tests: [] diff --git a/cmake/download_archive.cmake b/cmake/download_archive.cmake new file mode 100644 index 00000000000..820aafafb7e --- /dev/null +++ b/cmake/download_archive.cmake @@ -0,0 +1,40 @@ +# Copyright 2021 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set(_download_archive_TEMPORARY_DIR ${CMAKE_BINARY_DIR}/http_archives) +file(MAKE_DIRECTORY ${_download_archive_TEMPORARY_DIR}) + +# This is basically Bazel's http_archive. +# Note that strip_prefix strips the directory path prefix of the extracted +# archive content, and it may strip multiple directories. +function(download_archive destination url hash strip_prefix) + # Fetch and validate + set(_TEMPORARY_FILE ${_download_archive_TEMPORARY_DIR}/${strip_prefix}.tar.gz) + message(STATUS "Downloading from ${url}, if failed, please try configuring again") + file(DOWNLOAD ${url} ${_TEMPORARY_FILE} + TIMEOUT 60 + EXPECTED_HASH SHA256=${hash} + TLS_VERIFY ON) + # Extract + execute_process(COMMAND + ${CMAKE_COMMAND} -E tar xvf ${_TEMPORARY_FILE} + WORKING_DIRECTORY ${_download_archive_TEMPORARY_DIR} + OUTPUT_QUIET) + get_filename_component(_download_archive_Destination_Path ${destination} DIRECTORY) + file(MAKE_DIRECTORY ${_download_archive_Destination_Path}) + file(RENAME ${_download_archive_TEMPORARY_DIR}/${strip_prefix} ${destination}) + # Clean up + file(REMOVE ${_download_archive_TEMPORARY_DIR}/${strip_prefix}) + file(REMOVE ${_TEMPORARY_FILE}) +endfunction() diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 6ec36e6abfa..fa001849ba0 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -27,14 +27,23 @@ proto_re = re.compile('(.*)\\.proto') lib_map = {lib.name: lib for lib in libs} + third_party_proto_prefixes = {lib.proto_prefix for lib in external_proto_libraries} + + def third_party_proto_import_path(path): + """Removes third_party prefix to match ProtoBuf's relative import path.""" + for prefix in third_party_proto_prefixes: + if path.startswith(prefix): + return path[len(prefix):] + return path + def proto_replace_ext(filename, ext): m = proto_re.match(filename) if not m: return filename - return '${_gRPC_PROTO_GENS_DIR}/' + m.group(1) + ext + return '${_gRPC_PROTO_GENS_DIR}/' + third_party_proto_import_path(m.group(1)) + ext def is_absl_lib(lib_name): - return lib_name.startswith("absl/"); + return lib_name.startswith("absl/") def get_absl_dep(lib_name): return lib_map[lib_name].cmake_target @@ -356,11 +365,34 @@ include(cmake/upb.cmake) include(cmake/xxhash.cmake) include(cmake/zlib.cmake) + include(cmake/download_archive.cmake) + + % for external_proto_library in external_proto_libraries: + % if len(external_proto_library.urls) > 1: + # Setup external proto library at ${external_proto_library.destination} with ${len(external_proto_library.urls)} download URLs + % else: + # Setup external proto library at ${external_proto_library.destination} if it doesn't exist + % endif + % for download_url in external_proto_library.urls: + if (NOT EXISTS <%text>${CMAKE_CURRENT_SOURCE_DIR}/${external_proto_library.destination}) + # Download the archive via HTTP, validate the checksum, and extract to ${external_proto_library.destination}. + download_archive( + <%text>${CMAKE_CURRENT_SOURCE_DIR}/${external_proto_library.destination} + ${download_url} + ${external_proto_library.hash} + ${external_proto_library.strip_prefix} + ) + endif() + % endfor + % endfor if(WIN32) set(_gRPC_BASELIB_LIBRARIES ws2_32 crypt32) endif() + # Create directory for proto source files + set(_gRPC_PROTO_SRCS_DIR <%text>${CMAKE_BINARY_DIR}/protos) + file(MAKE_DIRECTORY <%text>${_gRPC_PROTO_SRCS_DIR}) # Create directory for generated .proto files set(_gRPC_PROTO_GENS_DIR <%text>${CMAKE_BINARY_DIR}/gens) file(MAKE_DIRECTORY <%text>${_gRPC_PROTO_GENS_DIR}) @@ -368,6 +400,10 @@ # protobuf_generate_grpc_cpp # -------------------------- # + # This method is no longer used by gRPC's CMake build process. However, it + # is used by many open source dependencies, that we might want to keep + # backward compatibility here. + # # Add custom commands to process ``.proto`` files to C++ using protoc and # GRPC plugin:: # @@ -382,40 +418,90 @@ return() endif() - set(_protobuf_include_path -I . -I <%text>${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}) foreach(FIL <%text>${ARGN}) - get_filename_component(ABS_FIL <%text>${FIL} ABSOLUTE) - get_filename_component(FIL_WE <%text>${FIL} NAME_WE) - file(RELATIVE_PATH REL_FIL <%text>${CMAKE_CURRENT_SOURCE_DIR} <%text>${ABS_FIL}) - get_filename_component(REL_DIR <%text>${REL_FIL} DIRECTORY) - set(RELFIL_WE "<%text>${REL_DIR}/${FIL_WE}") - - #if cross-compiling, find host plugin - if(CMAKE_CROSSCOMPILING) - find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin) - else() - set(_gRPC_CPP_PLUGIN $) - endif() - - add_custom_command( - OUTPUT <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" - <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" - <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" - <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" - <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" - COMMAND <%text>${_gRPC_PROTOBUF_PROTOC_EXECUTABLE} - ARGS --grpc_out=<%text>generate_mock_code=true:${_gRPC_PROTO_GENS_DIR} - --cpp_out=<%text>${_gRPC_PROTO_GENS_DIR} - --plugin=protoc-gen-grpc=<%text>${_gRPC_CPP_PLUGIN} - <%text>${_protobuf_include_path} - <%text>${REL_FIL} - DEPENDS <%text>${ABS_FIL} <%text>${_gRPC_PROTOBUF_PROTOC} <%text>${_gRPC_CPP_PLUGIN} - WORKING_DIRECTORY <%text>${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Running gRPC C++ protocol buffer compiler on <%text>${FIL}" - VERBATIM) + protobuf_generate_grpc_cpp_with_import_path_correction(<%text>${FIL} <%text>${FIL}) endforeach() endfunction() + # protobuf_generate_grpc_cpp_with_import_path_correction + # -------------------------- + # + # Add custom commands to process ``.proto`` files to C++ using protoc and + # GRPC plugin:: + # + # protobuf_generate_grpc_cpp_with_import_path_correction + # + # ``FILE_LOCATION`` + # The relative path of the ``.proto`` file to the project root + # ``IMPORT_PATH`` + # The proto import path that itself expected to be placed in. For + # example, a "bar.proto" file wants to be imported as + # `import "foo/bar.proto"`. Then we should place it under + # "/foo/bar.proto" instead of + # "/third_party/foo/bar.proto". This ensures + # correct symbol being generated and C++ include path being correct. + # More info can be found at https://github.com/grpc/grpc/pull/25272. + # + function(protobuf_generate_grpc_cpp_with_import_path_correction FILE_LOCATION IMPORT_PATH) + if(NOT FILE_LOCATION) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") + return() + endif() + + # Sets the include path for ProtoBuf files + set(_protobuf_include_path -I . -I <%text>${_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR}) + # The absolute path of the expected place for the input proto file + # For example, health proto has package name grpc.health.v1, it's expected to be: + # <%text>${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1/health.proto + get_filename_component(ABS_FIL <%text>${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH} ABSOLUTE) + # Get the name of the file, which used to generate output file names for + # this command. + # Example: "health" for "health.proto" + get_filename_component(FIL_WE <%text>${_gRPC_PROTO_SRCS_DIR}/${IMPORT_PATH} NAME_WE) + # Get the relative path between the expected place for the proto and the + # working directory. In normal cases, it would be equal IMPORT_PATH, but + # it's better to be agnostic to all the global folder locations (like the + # centralized location <%text>${_gRPC_PROTO_SRCS_DIR}). + # Example: grpc/health/v1/health.proto + file(RELATIVE_PATH REL_FIL <%text>${_gRPC_PROTO_SRCS_DIR} <%text>${ABS_FIL}) + # Get the directory of the relative path. + # Example: grpc/health/v1 + get_filename_component(REL_DIR <%text>${REL_FIL} DIRECTORY) + # Get the directory and name for output filenames generation. + # Example: "grpc/health/v1/health", the file name extension is omitted. + set(RELFIL_WE "<%text>${REL_DIR}/${FIL_WE}") + # Copy the proto file to a centralized location, with the correct import + # path. For example, health proto has package name grpc.health.v1, the bash + # equivalent would be: + # cp src/proto/grpc/health/v1/health.proto <%text>${_gRPC_PROTO_SRCS_DIR}/grpc/health/v1 + file(MAKE_DIRECTORY <%text>${_gRPC_PROTO_SRCS_DIR}/${REL_DIR}) + file(COPY <%text>${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION} DESTINATION <%text>${_gRPC_PROTO_SRCS_DIR}/${REL_DIR}) + + #if cross-compiling, find host plugin + if(CMAKE_CROSSCOMPILING) + find_program(_gRPC_CPP_PLUGIN grpc_cpp_plugin) + else() + set(_gRPC_CPP_PLUGIN $) + endif() + + add_custom_command( + OUTPUT <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" + <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" + <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" + <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" + <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" + COMMAND <%text>${_gRPC_PROTOBUF_PROTOC_EXECUTABLE} + ARGS --grpc_out=<%text>generate_mock_code=true:${_gRPC_PROTO_GENS_DIR} + --cpp_out=<%text>${_gRPC_PROTO_GENS_DIR} + --plugin=protoc-gen-grpc=<%text>${_gRPC_CPP_PLUGIN} + <%text>${_protobuf_include_path} + <%text>${REL_FIL} + DEPENDS <%text>${CMAKE_CURRENT_SOURCE_DIR}/${FILE_LOCATION} <%text>${ABS_FIL} <%text>${_gRPC_PROTOBUF_PROTOC} grpc_cpp_plugin + WORKING_DIRECTORY <%text>${_gRPC_PROTO_SRCS_DIR} + COMMENT "Running gRPC C++ protocol buffer compiler for <%text>${IMPORT_PATH}" + VERBATIM) + endfunction() + # These options allow users to enable or disable the building of the various # protoc plugins. For example, running CMake with # -DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF will disable building the C# plugin. @@ -455,8 +541,8 @@ DEPENDS tools_c tools_cxx) % for src in sorted(protobuf_gen_files): - protobuf_generate_grpc_cpp( - ${src} + protobuf_generate_grpc_cpp_with_import_path_correction( + ${src} ${third_party_proto_import_path(src)} ) % endfor diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat index 1948d26dab9..1ca38580359 100644 --- a/test/distrib/cpp/run_distrib_test_cmake.bat +++ b/test/distrib/cpp/run_distrib_test_cmake.bat @@ -68,7 +68,11 @@ popd @rem Just before installing gRPC, wipe out contents of all the submodules to simulate @rem a standalone build from an archive -git submodule deinit --all --force +@rem NOTE(lidiz) We used to use "git submodule deinit", but it leaves an empty +@rem folder for deinit-ed submodules, blocking the CMake download. For users +@rem downloaded gRPC code as an archive, they won't have submodule residual +@rem folders, like the following command trying to imitate. +git submodule foreach bash -c "cd $toplevel; rm -rf $name" @rem Install gRPC mkdir cmake\build diff --git a/tools/buildgen/extract_metadata_from_bazel_xml.py b/tools/buildgen/extract_metadata_from_bazel_xml.py index 396f635fb79..8af3e7b4ed9 100755 --- a/tools/buildgen/extract_metadata_from_bazel_xml.py +++ b/tools/buildgen/extract_metadata_from_bazel_xml.py @@ -30,11 +30,11 @@ # format entirely or simplify it to a point where it becomes self-explanatory # and doesn't need any detailed documentation. -import collections +from dataclasses import asdict +from dataclasses import dataclass +from dataclasses import field import os -import re import subprocess -import sys from typing import Any, Dict, Iterable, List, Optional import xml.etree.ElementTree as ET @@ -45,6 +45,48 @@ BuildMetadata = Dict[str, Any] BuildDict = Dict[str, BuildMetadata] BuildYaml = Dict[str, Any] +BuildMetadata = Dict[str, Any] +BuildDict = Dict[str, BuildMetadata] +BuildYaml = Dict[str, Any] + + +# This is basically a Python dict with predefined fields and types +@dataclass() +class ExternalProtoLibrary: + # The relative path of this proto library should be. Preferably, it should + # match the submodule path. + destination: str + # The prefix to remove in order to insure the proto import is correct. For + # more info, see description of https://github.com/grpc/grpc/pull/25272. + proto_prefix: str + # Following 3 fields should be filled by build metadata from Bazel. + urls: List[str] = field(default_factory=list) + hash: str = '' + strip_prefix: str = '' + + +EXTERNAL_PROTO_LIBRARIES = { + 'envoy_api': + ExternalProtoLibrary(destination='third_party/envoy-api', + proto_prefix='third_party/envoy-api/'), + 'com_google_googleapis': + ExternalProtoLibrary(destination='third_party/googleapis', + proto_prefix='third_party/googleapis/'), + 'com_github_cncf_udpa': + ExternalProtoLibrary(destination='third_party/xds', + proto_prefix='third_party/xds/'), + 'opencensus_proto': + ExternalProtoLibrary(destination='third_party/opencensus-proto/src', + proto_prefix='third_party/opencensus-proto/src/'), +} + + +def _maybe_get_internal_path(name: str) -> Optional[str]: + for key in EXTERNAL_PROTO_LIBRARIES: + if name.startswith('@' + key): + return key + return None + def _bazel_query_xml_tree(query: str) -> ET.Element: """Get xml output of bazel query invocation, parsed as XML tree""" @@ -98,6 +140,7 @@ def _extract_rules_from_bazel_xml(xml_tree): 'cc_binary', 'cc_test', 'cc_proto_library', + 'cc_proto_gen_validate', 'proto_library', 'upb_proto_library', 'upb_proto_reflection_library', @@ -109,6 +152,8 @@ def _extract_rules_from_bazel_xml(xml_tree): def _get_bazel_label(target_name: str) -> str: + if target_name.startswith('@'): + return target_name if ':' in target_name: return '//%s' % target_name else: @@ -149,17 +194,40 @@ def _extract_nonpublic_headers(bazel_rule: BuildMetadata) -> List[str]: def _extract_sources(bazel_rule: BuildMetadata) -> List[str]: """Gets list of source files from a bazel rule""" result = [] - for dep in bazel_rule['srcs']: - if dep.startswith('//') and (dep.endswith('.cc') or dep.endswith('.c') - or dep.endswith('.proto')): - result.append(_extract_source_file_path(dep)) + for src in bazel_rule['srcs']: + if src.endswith('.cc') or src.endswith('.c') or src.endswith('.proto'): + if src.startswith('//'): + # This source file is local to gRPC + result.append(_extract_source_file_path(src)) + else: + # This source file is external, and we need to translate the + # @REPO_NAME to a valid path prefix. At this stage, we need + # to check repo name, since the label/path mapping is not + # available in BUILD files. + external_proto_library_name = _maybe_get_internal_path(src) + if external_proto_library_name is not None: + result.append( + src.replace( + f'@{external_proto_library_name}//', + EXTERNAL_PROTO_LIBRARIES[ + external_proto_library_name].proto_prefix). + replace(':', '/')) return list(sorted(result)) def _extract_deps(bazel_rule: BuildMetadata, bazel_rules: BuildDict) -> List[str]: """Gets list of deps from from a bazel rule""" - return list(sorted(bazel_rule['deps'])) + deps = set(bazel_rule['deps']) + for src in bazel_rule['srcs']: + if not src.endswith('.cc') and not src.endswith( + '.c') and not src.endswith('.proto'): + if src in bazel_rules: + # This label doesn't point to a source file, but another Bazel + # target. This is required for :pkg_cc_proto_validate targets, + # and it's generally allowed by Bazel. + deps.add(src) + return list(sorted(list(deps))) def _create_target_from_bazel_rule(target_name: str, @@ -262,7 +330,6 @@ def _compute_transitive_metadata( # This item is not processed before, compute now _compute_transitive_metadata(dep, bazel_rules, bazel_label_to_dep_name) - transitive_deps.update(bazel_rules[dep].get( '_TRANSITIVE_DEPS', [])) collapsed_deps.update( @@ -336,15 +403,16 @@ def _compute_transitive_metadata( bazel_rule['_EXCLUDE_DEPS'] = list(sorted(exclude_deps)) -# TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a slightly different logic) +# TODO(jtattermusch): deduplicate with transitive_dependencies.py (which has a +# slightly different logic) # TODO(jtattermusch): This is done to avoid introducing too many intermediate # libraries into the build.yaml-based builds (which might in cause issues -# building language-specific artifacts) and also because the libraries -# in build.yaml-based build are generally considered units of distributions -# (= public libraries that are visible to the user and are installable), -# while in bazel builds it is customary to define larger number of smaller -# "sublibraries". The need for elision (and expansion) -# of intermediate libraries can be re-evaluated in the future. +# building language-specific artifacts) and also because the libraries in +# build.yaml-based build are generally considered units of distributions (= +# public libraries that are visible to the user and are installable), while in +# bazel builds it is customary to define larger number of smaller +# "sublibraries". The need for elision (and expansion) of intermediate libraries +# can be re-evaluated in the future. def _populate_transitive_metadata(bazel_rules: Any, public_dep_names: Iterable[str]) -> None: """Add 'transitive_deps' field for each of the rules""" @@ -722,6 +790,48 @@ def _generate_build_extra_metadata_for_tests( return test_metadata +def _parse_http_archives(xml_tree: ET.Element) -> List[ExternalProtoLibrary]: + """Parse Bazel http_archive rule into ExternalProtoLibrary objects.""" + result = [] + for xml_http_archive in xml_tree: + if xml_http_archive.tag != 'rule' or xml_http_archive.attrib[ + 'class'] != 'http_archive': + continue + # A distilled Python representation of Bazel http_archive + http_archive = dict() + for xml_node in xml_http_archive: + if xml_node.attrib['name'] == 'name': + http_archive["name"] = xml_node.attrib['value'] + if xml_node.attrib['name'] == 'urls': + http_archive["urls"] = [] + for url_node in xml_node: + http_archive["urls"].append(url_node.attrib['value']) + if xml_node.attrib['name'] == 'url': + http_archive["urls"] = [xml_node.attrib['value']] + if xml_node.attrib['name'] == 'sha256': + http_archive["hash"] = xml_node.attrib['value'] + if xml_node.attrib['name'] == 'strip_prefix': + http_archive["strip_prefix"] = xml_node.attrib['value'] + if http_archive["name"] not in EXTERNAL_PROTO_LIBRARIES: + # If this http archive is not one of the external proto libraries, + # we don't want to include it as a CMake target + continue + lib = EXTERNAL_PROTO_LIBRARIES[http_archive["name"]] + lib.urls = http_archive["urls"] + lib.hash = http_archive["hash"] + lib.strip_prefix = http_archive["strip_prefix"] + result.append(lib) + return result + + +def _generate_external_proto_libraries() -> List[Dict[str, Any]]: + """Generates the build metadata for external proto libraries""" + xml_tree = _bazel_query_xml_tree('kind(http_archive, //external:*)') + libraries = _parse_http_archives(xml_tree) + libraries.sort(key=lambda x: x.destination) + return list(map(asdict, libraries)) + + def _detect_and_print_issues(build_yaml_like: BuildYaml) -> None: """Try detecting some unusual situations and warn about them.""" for tgt in build_yaml_like['targets']: @@ -1103,6 +1213,15 @@ all_targets_dict = _generate_build_metadata(all_extra_metadata, bazel_rules) # 'tests': { TARGET_DICT_FOR_TEST_XYZ, ...} } build_yaml_like = _convert_to_build_yaml_like(all_targets_dict) +# Step 7: generates build metadata for external ProtoBuf libraries. +# We only want the ProtoBuf sources from these ProtoBuf dependencies, which may +# not be present in our release source tar balls. These rules will be used in CMake +# to download these libraries if not existed. Even if the download failed, it +# will be a soft error that doesn't block existing target from successfully +# built. +build_yaml_like[ + 'external_proto_libraries'] = _generate_external_proto_libraries() + # detect and report some suspicious situations we've seen before _detect_and_print_issues(build_yaml_like) diff --git a/tools/buildgen/plugins/check_attrs.py b/tools/buildgen/plugins/check_attrs.py index 34729ed1686..14a59401493 100644 --- a/tools/buildgen/plugins/check_attrs.py +++ b/tools/buildgen/plugins/check_attrs.py @@ -89,6 +89,13 @@ VALID_ATTRIBUTE_KEYS_MAP = { 'vs_proj_dir': anything(), 'zlib': one_of((True,)), }, + 'external_proto_library': { + 'destination': anything(), + 'proto_prefix': anything(), + 'urls': anything(), + 'hash': anything(), + 'strip_prefix': anything(), + } } @@ -113,10 +120,10 @@ def check_attributes(entity, kind, errors): def mako_plugin(dictionary): """The exported plugin code for check_attr. - This validates that filegroups, libs, and target can have only valid - attributes. This is mainly for preventing build.yaml from having - unnecessary and misleading attributes accidentally. - """ + This validates that filegroups, libs, and target can have only valid + attributes. This is mainly for preventing build.yaml from having + unnecessary and misleading attributes accidentally. + """ errors = [] for filegroup in dictionary.get('filegroups', {}): @@ -125,5 +132,7 @@ def mako_plugin(dictionary): check_attributes(lib, 'lib', errors) for target in dictionary.get('targets', {}): check_attributes(target, 'target', errors) + for target in dictionary.get('external_proto_libraries', {}): + check_attributes(target, 'external_proto_library', errors) if errors: raise Exception('\n'.join(errors))