diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12f46d181e6..c28eb974f88 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # repository as the source of truth for module ownership. /**/OWNERS @markdroth @nicolasnoble @a11r /bazel/** @nicolasnoble @dgquintas @a11r @vjpai -/cmake/** @jtattermusch @nicolasnoble @matt-kwong +/cmake/** @jtattermusch @nicolasnoble @mehrdada /src/core/ext/filters/client_channel/** @markdroth @dgquintas @AspirinSJL -/tools/dockerfile/** @jtattermusch @matt-kwong @nicolasnoble -/tools/run_tests/performance/** @ncteisen @matt-kwong @jtattermusch +/tools/dockerfile/** @jtattermusch @mehrdada @nicolasnoble +/tools/run_tests/performance/** @ncteisen @apolcyn @jtattermusch diff --git a/.github/mergeable.yml b/.github/mergeable.yml new file mode 100644 index 00000000000..30692095c46 --- /dev/null +++ b/.github/mergeable.yml @@ -0,0 +1,14 @@ +mergeable: + pull_requests: + label: + or: + - and: + - must_include: + regex: 'release notes: yes' + message: 'Please include release note: yes' + - must_include: + regex: '^lang\/' + message: 'Please include a language label' + - must_include: + regex: 'release notes: no' + message: 'Please include release note: no' diff --git a/BUILD b/BUILD index ee4b5dfaecc..b59170e05eb 100644 --- a/BUILD +++ b/BUILD @@ -485,7 +485,7 @@ grpc_cc_library( grpc_cc_library( name = "census", srcs = [ - "src/cpp/ext/filters/census/grpc_context.cc", + "src/core/ext/filters/census/grpc_context.cc", ], language = "c++", public_hdrs = [ @@ -560,6 +560,7 @@ grpc_cc_library( "src/core/lib/gprpp/fork.h", "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/memory.h", + "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h", ], @@ -1010,6 +1011,7 @@ grpc_cc_library( "src/core/lib/iomgr/cfstream_handle.cc", "src/core/lib/iomgr/endpoint_cfstream.cc", "src/core/lib/iomgr/error_cfstream.cc", + "src/core/lib/iomgr/iomgr_posix_cfstream.cc", "src/core/lib/iomgr/tcp_client_cfstream.cc", ], hdrs = [ @@ -1433,8 +1435,11 @@ grpc_cc_library( "src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc", ], hdrs = [ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h", @@ -1496,6 +1501,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.cc", "src/core/lib/security/credentials/ssl/ssl_credentials.cc", "src/core/lib/security/security_connector/alts_security_connector.cc", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/security_connector.cc", "src/core/lib/security/transport/client_auth_filter.cc", @@ -1524,6 +1531,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", @@ -1877,8 +1886,8 @@ grpc_cc_library( "src/core/tsi/alts/handshaker/alts_tsi_handshaker_private.h", "src/core/tsi/alts/handshaker/alts_tsi_utils.h", "src/core/tsi/alts_transport_security.h", - "src/core/tsi/local_transport_security.h", "src/core/tsi/fake_transport_security.h", + "src/core/tsi/local_transport_security.h", "src/core/tsi/ssl/session_cache/ssl_session.h", "src/core/tsi/ssl/session_cache/ssl_session_cache.h", "src/core/tsi/ssl_transport_security.h", @@ -2121,7 +2130,7 @@ grpc_cc_library( "src/cpp/ext/filters/census/channel_filter.cc", "src/cpp/ext/filters/census/client_filter.cc", "src/cpp/ext/filters/census/context.cc", - "src/cpp/ext/filters/census/grpc_context.cc", + "src/core/ext/filters/census/grpc_context.cc", "src/cpp/ext/filters/census/grpc_plugin.cc", "src/cpp/ext/filters/census/measures.cc", "src/cpp/ext/filters/census/rpc_encoding.cc", diff --git a/CMakeLists.txt b/CMakeLists.txt index 84e9c08cb5b..32d7a9b94d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,8 @@ if (MSVC) add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later @@ -298,6 +300,7 @@ add_dependencies(buildtests_c grpc_completion_queue_test) add_dependencies(buildtests_c grpc_completion_queue_threading_test) add_dependencies(buildtests_c grpc_credentials_test) add_dependencies(buildtests_c grpc_fetch_oauth2) +add_dependencies(buildtests_c grpc_ipv6_loopback_available_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c grpc_json_token_test) endif() @@ -578,6 +581,7 @@ add_dependencies(buildtests_cxx generic_end2end_test) add_dependencies(buildtests_cxx golden_file_test) add_dependencies(buildtests_cxx grpc_alts_credentials_options_test) add_dependencies(buildtests_cxx grpc_cli) +add_dependencies(buildtests_cxx grpc_linux_system_roots_test) add_dependencies(buildtests_cxx grpc_tool_test) add_dependencies(buildtests_cxx grpclb_api_test) add_dependencies(buildtests_cxx grpclb_end2end_test) @@ -659,27 +663,17 @@ add_dependencies(buildtests_cxx transport_security_common_api_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test_unsecure) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test) -endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx address_sorting_test_unsecure) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx address_sorting_test) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx cancel_ares_query_test) -endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -1136,6 +1130,8 @@ add_library(grpc src/core/lib/security/credentials/plugin/plugin_credentials.cc src/core/lib/security/credentials/ssl/ssl_credentials.cc src/core/lib/security/security_connector/alts_security_connector.cc + src/core/lib/security/security_connector/load_system_roots_fallback.cc + src/core/lib/security/security_connector/load_system_roots_linux.cc src/core/lib/security/security_connector/local_security_connector.cc src/core/lib/security/security_connector/security_connector.cc src/core/lib/security/transport/client_auth_filter.cc @@ -1236,11 +1232,14 @@ add_library(grpc src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc - src/cpp/ext/filters/census/grpc_context.cc + src/core/ext/filters/census/grpc_context.cc src/core/ext/filters/max_age/max_age_filter.cc src/core/ext/filters/message_size/message_size_filter.cc src/core/ext/filters/http/client_authority_filter.cc @@ -1563,6 +1562,8 @@ add_library(grpc_cronet src/core/lib/security/credentials/plugin/plugin_credentials.cc src/core/lib/security/credentials/ssl/ssl_credentials.cc src/core/lib/security/security_connector/alts_security_connector.cc + src/core/lib/security/security_connector/load_system_roots_fallback.cc + src/core/lib/security/security_connector/load_system_roots_linux.cc src/core/lib/security/security_connector/local_security_connector.cc src/core/lib/security/security_connector/security_connector.cc src/core/lib/security/transport/client_auth_filter.cc @@ -2538,8 +2539,11 @@ add_library(grpc_unsecure src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc @@ -2556,7 +2560,7 @@ add_library(grpc_unsecure third_party/nanopb/pb_encode.c src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc - src/cpp/ext/filters/census/grpc_context.cc + src/core/ext/filters/census/grpc_context.cc src/core/ext/filters/max_age/max_age_filter.cc src/core/ext/filters/message_size/message_size_filter.cc src/core/ext/filters/http/client_authority_filter.cc @@ -3344,7 +3348,7 @@ add_library(grpc++_cronet src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc src/core/ext/transport/chttp2/server/chttp2_server.cc - src/cpp/ext/filters/census/grpc_context.cc + src/core/ext/filters/census/grpc_context.cc ) if(WIN32 AND MSVC) @@ -7323,6 +7327,35 @@ target_link_libraries(grpc_fetch_oauth2 gpr ) +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + +add_executable(grpc_ipv6_loopback_available_test + test/core/iomgr/grpc_ipv6_loopback_available_test.cc +) + + +target_include_directories(grpc_ipv6_loopback_available_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} +) + +target_link_libraries(grpc_ipv6_loopback_available_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -12118,6 +12151,44 @@ if (gRPC_INSTALL) endif() endif (gRPC_BUILD_CODEGEN) +if (gRPC_BUILD_TESTS) + +add_executable(grpc_linux_system_roots_test + test/core/security/linux_system_roots_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(grpc_linux_system_roots_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_linux_system_roots_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_CODEGEN) add_executable(grpc_node_plugin @@ -16179,7 +16250,6 @@ target_link_libraries(inproc_nosec_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test_unsecure test/cpp/naming/resolver_component_test.cc @@ -16219,10 +16289,8 @@ target_link_libraries(resolver_component_test_unsecure ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test test/cpp/naming/resolver_component_test.cc @@ -16262,7 +16330,6 @@ target_link_libraries(resolver_component_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -16351,7 +16418,6 @@ target_link_libraries(resolver_component_tests_runner_invoker endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(address_sorting_test_unsecure test/cpp/naming/address_sorting_test.cc @@ -16391,10 +16457,8 @@ target_link_libraries(address_sorting_test_unsecure ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(address_sorting_test test/cpp/naming/address_sorting_test.cc @@ -16434,10 +16498,8 @@ target_link_libraries(address_sorting_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(cancel_ares_query_test test/cpp/naming/cancel_ares_query_test.cc @@ -16477,7 +16539,6 @@ target_link_libraries(cancel_ares_query_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 2b4ed146ccc..3454ef85870 100644 --- a/Makefile +++ b/Makefile @@ -767,11 +767,20 @@ else LDLIBS_SECURE += $(addprefix -l, $(LIBS_SECURE)) endif +# gpr .pc file +PC_NAME = gpr +PC_DESCRIPTION = gRPC platform support library +PC_CFLAGS = +PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GPR) +PC_LIBS_PRIVATE = $(PC_LIBS_GPR) +PC_LIB = -lgpr +GPR_PC_FILE := $(CORE_PC_TEMPLATE) + # grpc .pc file PC_NAME = gRPC PC_DESCRIPTION = high performance general RPC framework PC_CFLAGS = -PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GRPC) $(PC_REQUIRES_SECURE) +PC_REQUIRES_PRIVATE = gpr $(PC_REQUIRES_GRPC) $(PC_REQUIRES_SECURE) PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) $(PC_LIBS_SECURE) PC_LIB = -lgrpc GRPC_PC_FILE := $(CORE_PC_TEMPLATE) @@ -780,7 +789,7 @@ GRPC_PC_FILE := $(CORE_PC_TEMPLATE) PC_NAME = gRPC unsecure PC_DESCRIPTION = high performance general RPC framework without SSL PC_CFLAGS = -PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GRPC) +PC_REQUIRES_PRIVATE = gpr $(PC_REQUIRES_GRPC) PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) PC_LIB = -lgrpc GRPC_UNSECURE_PC_FILE := $(CORE_PC_TEMPLATE) @@ -1022,6 +1031,7 @@ grpc_completion_queue_threading_test: $(BINDIR)/$(CONFIG)/grpc_completion_queue_ grpc_create_jwt: $(BINDIR)/$(CONFIG)/grpc_create_jwt grpc_credentials_test: $(BINDIR)/$(CONFIG)/grpc_credentials_test grpc_fetch_oauth2: $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 +grpc_ipv6_loopback_available_test: $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test grpc_json_token_test: $(BINDIR)/$(CONFIG)/grpc_json_token_test grpc_jwt_verifier_test: $(BINDIR)/$(CONFIG)/grpc_jwt_verifier_test grpc_print_google_default_creds_token: $(BINDIR)/$(CONFIG)/grpc_print_google_default_creds_token @@ -1169,6 +1179,7 @@ grpc_alts_credentials_options_test: $(BINDIR)/$(CONFIG)/grpc_alts_credentials_op grpc_cli: $(BINDIR)/$(CONFIG)/grpc_cli grpc_cpp_plugin: $(BINDIR)/$(CONFIG)/grpc_cpp_plugin grpc_csharp_plugin: $(BINDIR)/$(CONFIG)/grpc_csharp_plugin +grpc_linux_system_roots_test: $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test grpc_node_plugin: $(BINDIR)/$(CONFIG)/grpc_node_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin grpc_php_plugin: $(BINDIR)/$(CONFIG)/grpc_php_plugin @@ -1381,6 +1392,8 @@ static_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libaddress_sorting.a static_cxx: pc_cxx pc_cxx_unsecure cache.mk $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc++_cronet.a $(LIBDIR)/$(CONFIG)/libgrpc++_error_details.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_unsecure.a $(LIBDIR)/$(CONFIG)/libgrpcpp_channelz.a +static_csharp: static_c $(LIBDIR)/$(CONFIG)/libgrpc_csharp_ext.a + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)address_sorting$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)gpr$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_cronet$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) $(LIBDIR)/$(CONFIG)/$(SHARED_PREFIX)grpc_unsecure$(SHARED_VERSION_CORE).$(SHARED_EXT_CORE) @@ -1394,9 +1407,9 @@ plugins: $(PROTOC_PLUGINS) privatelibs: privatelibs_c privatelibs_cxx privatelibs_c: $(LIBDIR)/$(CONFIG)/libalts_test_util.a $(LIBDIR)/$(CONFIG)/libcxxabi.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util_unsecure.a $(LIBDIR)/$(CONFIG)/libreconnect_server.a $(LIBDIR)/$(CONFIG)/libtest_tcp_server.a $(LIBDIR)/$(CONFIG)/libz.a $(LIBDIR)/$(CONFIG)/libares.a $(LIBDIR)/$(CONFIG)/libbad_client_test.a $(LIBDIR)/$(CONFIG)/libbad_ssl_test_server.a $(LIBDIR)/$(CONFIG)/libend2end_tests.a $(LIBDIR)/$(CONFIG)/libend2end_nosec_tests.a -pc_c: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc +pc_c: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc -pc_c_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc +pc_c_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc @@ -1470,6 +1483,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test \ $(BINDIR)/$(CONFIG)/grpc_credentials_test \ $(BINDIR)/$(CONFIG)/grpc_fetch_oauth2 \ + $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test \ $(BINDIR)/$(CONFIG)/grpc_json_token_test \ $(BINDIR)/$(CONFIG)/grpc_jwt_verifier_test \ $(BINDIR)/$(CONFIG)/grpc_security_connector_test \ @@ -1666,6 +1680,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ @@ -1845,6 +1860,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ @@ -2026,6 +2042,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/grpc_completion_queue_threading_test || ( echo test grpc_completion_queue_threading_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_credentials_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_credentials_test || ( echo test grpc_credentials_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_ipv6_loopback_available_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test || ( echo test grpc_ipv6_loopback_available_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_json_token_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_json_token_test || ( echo test grpc_json_token_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_jwt_verifier_test" @@ -2310,6 +2328,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/golden_file_test || ( echo test golden_file_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_alts_credentials_options_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test || ( echo test grpc_alts_credentials_options_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_linux_system_roots_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test || ( echo test grpc_linux_system_roots_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_tool_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_tool_test || ( echo test grpc_tool_test failed ; exit 1 ) $(E) "[RUN] Testing grpclb_api_test" @@ -2508,6 +2528,11 @@ cache.mk:: $(E) "[MAKE] Generating $@" $(Q) echo "$(CACHE_MK)" | tr , '\n' >$@ +$(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc: + $(E) "[MAKE] Generating $@" + $(Q) mkdir -p $(@D) + $(Q) echo "$(GPR_PC_FILE)" | tr , '\n' >$@ + $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc: $(E) "[MAKE] Generating $@" $(Q) mkdir -p $(@D) @@ -3118,6 +3143,7 @@ install-grpc-cli: grpc_cli install-pkg-config_c: pc_c pc_c_unsecure $(E) "[INSTALL] Installing C pkg-config files" $(Q) $(INSTALL) -d $(prefix)/lib/pkgconfig + $(Q) $(INSTALL) -m 0644 $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc $(prefix)/lib/pkgconfig/gpr.pc $(Q) $(INSTALL) -m 0644 $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc $(prefix)/lib/pkgconfig/grpc.pc $(Q) $(INSTALL) -m 0644 $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc $(prefix)/lib/pkgconfig/grpc_unsecure.pc @@ -3602,6 +3628,8 @@ LIBGRPC_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -3702,11 +3730,14 @@ LIBGRPC_SRC = \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ - src/cpp/ext/filters/census/grpc_context.cc \ + src/core/ext/filters/census/grpc_context.cc \ src/core/ext/filters/max_age/max_age_filter.cc \ src/core/ext/filters/message_size/message_size_filter.cc \ src/core/ext/filters/http/client_authority_filter.cc \ @@ -4028,6 +4059,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -4970,8 +5003,11 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ @@ -4988,7 +5024,7 @@ LIBGRPC_UNSECURE_SRC = \ third_party/nanopb/pb_encode.c \ src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc \ src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc \ - src/cpp/ext/filters/census/grpc_context.cc \ + src/core/ext/filters/census/grpc_context.cc \ src/core/ext/filters/max_age/max_age_filter.cc \ src/core/ext/filters/message_size/message_size_filter.cc \ src/core/ext/filters/http/client_authority_filter.cc \ @@ -5764,7 +5800,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc \ src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc \ src/core/ext/transport/chttp2/server/chttp2_server.cc \ - src/cpp/ext/filters/census/grpc_context.cc \ + src/core/ext/filters/census/grpc_context.cc \ PUBLIC_HEADERS_CXX += \ include/grpc++/alarm.h \ @@ -12363,6 +12399,38 @@ endif endif +GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_SRC = \ + test/core/iomgr/grpc_ipv6_loopback_available_test.cc \ + +GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test: $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/grpc_ipv6_loopback_available_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/iomgr/grpc_ipv6_loopback_available_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_ipv6_loopback_available_test: $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_IPV6_LOOPBACK_AVAILABLE_TEST_OBJS:.o=.dep) +endif +endif + + GRPC_JSON_TOKEN_TEST_SRC = \ test/core/security/json_token_test.cc \ @@ -17863,6 +17931,49 @@ ifneq ($(NO_DEPS),true) endif +GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC = \ + test/core/security/linux_system_roots_test.cc \ + +GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: $(PROTOBUF_DEP) $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/security/linux_system_roots_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_linux_system_roots_test: $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS:.o=.dep) +endif +endif + + GRPC_NODE_PLUGIN_SRC = \ src/compiler/node_plugin.cc \ @@ -24607,6 +24718,8 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/credentials/plugin/plugin_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/credentials/ssl/ssl_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/alts_security_connector.cc: $(OPENSSL_DEP) +src/core/lib/security/security_connector/load_system_roots_fallback.cc: $(OPENSSL_DEP) +src/core/lib/security/security_connector/load_system_roots_linux.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/local_security_connector.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/security_connector.cc: $(OPENSSL_DEP) src/core/lib/security/transport/client_auth_filter.cc: $(OPENSSL_DEP) diff --git a/WORKSPACE b/WORKSPACE index 8b68c0d2b19..9a0c41977f0 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,44 @@ -workspace(name = "com_github_grpc_grpc") +workspace(name="com_github_grpc_grpc") load("//bazel:grpc_deps.bzl", "grpc_deps", "grpc_test_only_deps") grpc_deps() grpc_test_only_deps() + +new_http_archive( + name="cython", + sha256="d68138a2381afbdd0876c3cb2a22389043fa01c4badede1228ee073032b07a27", + urls=[ + "https://github.com/cython/cython/archive/c2b80d87658a8525ce091cbe146cb7eaa29fed5c.tar.gz", + ], + strip_prefix="cython-c2b80d87658a8525ce091cbe146cb7eaa29fed5c", + build_file="//third_party:cython.BUILD", +) + +load("//third_party/py:python_configure.bzl", "python_configure") +python_configure(name="local_config_python") + +git_repository( + name="io_bazel_rules_python", + remote="https://github.com/bazelbuild/rules_python.git", + commit="8b5d0683a7d878b28fffe464779c8a53659fc645", +) + +load("@io_bazel_rules_python//python:pip.bzl", "pip_repositories", "pip_import") + +pip_repositories() +pip_import( + name="grpc_python_dependencies", + requirements="//:requirements.bazel.txt", +) + +load("@grpc_python_dependencies//:requirements.bzl", "pip_install") +pip_install() + +git_repository( + name="org_pubref_rules_protobuf", + remote="https://github.com/pubref/rules_protobuf", + tag="v0.8.2", +) + +load("@org_pubref_rules_protobuf//python:rules.bzl", "py_proto_repositories") +py_proto_repositories() diff --git a/bazel/cython_library.bzl b/bazel/cython_library.bzl new file mode 100644 index 00000000000..48b41d74e8c --- /dev/null +++ b/bazel/cython_library.bzl @@ -0,0 +1,74 @@ +"""Custom rules for gRPC Python""" + + +# Adapted with modifications from +# tensorflow/tensorflow/core/platform/default/build_config.bzl +# Native Bazel rules don't exist yet to compile Cython code, but rules have +# been written at cython/cython and tensorflow/tensorflow. We branch from +# Tensorflow's version as it is more actively maintained and works for gRPC +# Python's needs. +def pyx_library(name, deps=[], py_deps=[], srcs=[], **kwargs): + """Compiles a group of .pyx / .pxd / .py files. + + First runs Cython to create .cpp files for each input .pyx or .py + .pxd + pair. Then builds a shared object for each, passing "deps" to each cc_binary + rule (includes Python headers by default). Finally, creates a py_library rule + with the shared objects and any pure Python "srcs", with py_deps as its + dependencies; the shared objects can be imported like normal Python files. + + Args: + name: Name for the rule. + deps: C/C++ dependencies of the Cython (e.g. Numpy headers). + py_deps: Pure Python dependencies of the final library. + srcs: .py, .pyx, or .pxd files to either compile or pass through. + **kwargs: Extra keyword arguments passed to the py_library. + """ + # First filter out files that should be run compiled vs. passed through. + py_srcs = [] + pyx_srcs = [] + pxd_srcs = [] + for src in srcs: + if src.endswith(".pyx") or (src.endswith(".py") and + src[:-3] + ".pxd" in srcs): + pyx_srcs.append(src) + elif src.endswith(".py"): + py_srcs.append(src) + else: + pxd_srcs.append(src) + if src.endswith("__init__.py"): + pxd_srcs.append(src) + + # Invoke cython to produce the shared object libraries. + for filename in pyx_srcs: + native.genrule( + name=filename + "_cython_translation", + srcs=[filename], + outs=[filename.split(".")[0] + ".cpp"], + # Optionally use PYTHON_BIN_PATH on Linux platforms so that python 3 + # works. Windows has issues with cython_binary so skip PYTHON_BIN_PATH. + cmd= + "PYTHONHASHSEED=0 $(location @cython//:cython_binary) --cplus $(SRCS) --output-file $(OUTS)", + tools=["@cython//:cython_binary"] + pxd_srcs, + ) + + shared_objects = [] + for src in pyx_srcs: + stem = src.split(".")[0] + shared_object_name = stem + ".so" + native.cc_binary( + name=shared_object_name, + srcs=[stem + ".cpp"], + deps=deps + ["@local_config_python//:python_headers"], + linkshared=1, + ) + shared_objects.append(shared_object_name) + + # Now create a py_library with these shared objects as data. + native.py_library( + name=name, + srcs=py_srcs, + deps=py_deps, + srcs_version="PY2AND3", + data=shared_objects, + **kwargs) + diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 7c349dbcdcf..d7519b14532 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -116,9 +116,9 @@ def grpc_deps(): if "com_google_protobuf" not in native.existing_rules(): native.http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-b5fbb742af122b565925987e65c08957739976a7", - url = "https://github.com/google/protobuf/archive/b5fbb742af122b565925987e65c08957739976a7.tar.gz", - ) + strip_prefix = "protobuf-48cb18e5c419ddd23d9badcfe4e9df7bde1979b2", + url = "https://github.com/google/protobuf/archive/48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz", + ) if "com_github_nanopb_nanopb" not in native.existing_rules(): native.new_http_archive( diff --git a/build.yaml b/build.yaml index 30389ec1146..34bec009336 100644 --- a/build.yaml +++ b/build.yaml @@ -100,7 +100,7 @@ filegroups: public_headers: - include/grpc/census.h src: - - src/cpp/ext/filters/census/grpc_context.cc + - src/core/ext/filters/census/grpc_context.cc uses: - grpc_base - name: cmdline @@ -196,6 +196,7 @@ filegroups: - src/core/lib/gprpp/fork.h - src/core/lib/gprpp/manual_constructor.h - src/core/lib/gprpp/memory.h + - src/core/lib/gprpp/mutex_lock.h - src/core/lib/gprpp/thd.h - src/core/lib/profiling/timers.h uses: @@ -548,6 +549,7 @@ filegroups: - src/core/lib/iomgr/cfstream_handle.cc - src/core/lib/iomgr/endpoint_cfstream.cc - src/core/lib/iomgr/error_cfstream.cc + - src/core/lib/iomgr/iomgr_posix_cfstream.cc - src/core/lib/iomgr/tcp_client_cfstream.cc uses: - grpc_base_headers @@ -740,8 +742,11 @@ filegroups: - src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + - src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc plugin: grpc_resolver_dns_ares uses: - grpc_base @@ -789,6 +794,8 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.h - src/core/lib/security/credentials/ssl/ssl_credentials.h - src/core/lib/security/security_connector/alts_security_connector.h + - src/core/lib/security/security_connector/load_system_roots.h + - src/core/lib/security/security_connector/load_system_roots_linux.h - src/core/lib/security/security_connector/local_security_connector.h - src/core/lib/security/security_connector/security_connector.h - src/core/lib/security/transport/auth_filters.h @@ -816,6 +823,8 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.cc - src/core/lib/security/credentials/ssl/ssl_credentials.cc - src/core/lib/security/security_connector/alts_security_connector.cc + - src/core/lib/security/security_connector/load_system_roots_fallback.cc + - src/core/lib/security/security_connector/load_system_roots_linux.cc - src/core/lib/security/security_connector/local_security_connector.cc - src/core/lib/security/security_connector/security_connector.cc - src/core/lib/security/transport/client_auth_filter.cc @@ -2730,6 +2739,18 @@ targets: - grpc - gpr_test_util - gpr +- name: grpc_ipv6_loopback_available_test + build: test + language: c + src: + - test/core/iomgr/grpc_ipv6_loopback_available_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr + exclude_iomgrs: + - uv - name: grpc_json_token_test build: test language: c @@ -4091,6 +4112,7 @@ targets: - mac - linux - posix + uses_polling: false - name: bm_error build: test language: c++ @@ -4683,6 +4705,17 @@ targets: secure: false vs_config_type: Application vs_project_guid: '{3C813052-A49A-4662-B90A-1ADBEC7EE453}' +- name: grpc_linux_system_roots_test + gtest: true + build: test + language: c++ + src: + - test/core/security/linux_system_roots_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_node_plugin build: protoc language: c++ diff --git a/cmake/OWNERS b/cmake/OWNERS index 530a941c466..21981a7c555 100644 --- a/cmake/OWNERS +++ b/cmake/OWNERS @@ -1,4 +1,4 @@ set noparent @jtattermusch @nicolasnoble -@matt-kwong +@mehrdada diff --git a/cmake/gflags.cmake b/cmake/gflags.cmake index 01e0a75b60b..c301b1cdb6d 100644 --- a/cmake/gflags.cmake +++ b/cmake/gflags.cmake @@ -28,8 +28,8 @@ if("${gRPC_GFLAGS_PROVIDER}" STREQUAL "module") elseif("${gRPC_GFLAGS_PROVIDER}" STREQUAL "package") # Use "CONFIG" as there is no built-in cmake module for gflags. find_package(gflags REQUIRED CONFIG) - if(TARGET gflags::gflags) - set(_gRPC_GFLAGS_LIBRARIES gflags::gflags) + if(TARGET gflags) + set(_gRPC_GFLAGS_LIBRARIES gflags) set(_gRPC_GFLAGS_INCLUDE_DIR ${GFLAGS_INCLUDE_DIR}) endif() set(_gRPC_FIND_GFLAGS "if(NOT gflags_FOUND)\n find_package(gflags CONFIG)\nendif()") diff --git a/composer.json b/composer.json index 3e31baaa64e..b8f4346444f 100644 --- a/composer.json +++ b/composer.json @@ -9,7 +9,7 @@ "php": ">=5.5.0" }, "require-dev": { - "google/auth": "v0.9" + "google/auth": "^v1.3.0" }, "suggest": { "ext-protobuf": "For better performance, install the protobuf C extension.", diff --git a/config.m4 b/config.m4 index c277ccafc8c..7825274eea9 100644 --- a/config.m4 +++ b/config.m4 @@ -280,6 +280,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -380,11 +382,14 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ + src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc \ - src/cpp/ext/filters/census/grpc_context.cc \ + src/core/ext/filters/census/grpc_context.cc \ src/core/ext/filters/max_age/max_age_filter.cc \ src/core/ext/filters/message_size/message_size_filter.cc \ src/core/ext/filters/http/client_authority_filter.cc \ @@ -655,6 +660,7 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/php/ext/grpc) PHP_ADD_BUILD_DIR($ext_builddir/src/boringssl) + PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/census) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb) PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/filters/client_channel/lb_policy/grpclb/proto/grpc/lb/v1) @@ -718,7 +724,6 @@ if test "$PHP_GRPC" != "no"; then PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi/alts/handshaker) PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi/alts/zero_copy_frame_protector) PHP_ADD_BUILD_DIR($ext_builddir/src/core/tsi/ssl/session_cache) - PHP_ADD_BUILD_DIR($ext_builddir/src/cpp/ext/filters/census) PHP_ADD_BUILD_DIR($ext_builddir/third_party/address_sorting) PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto) PHP_ADD_BUILD_DIR($ext_builddir/third_party/boringssl/crypto/asn1) diff --git a/config.w32 b/config.w32 index 2857781dd57..a9d1e6c9d01 100644 --- a/config.w32 +++ b/config.w32 @@ -255,6 +255,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\security\\credentials\\plugin\\plugin_credentials.cc " + "src\\core\\lib\\security\\credentials\\ssl\\ssl_credentials.cc " + "src\\core\\lib\\security\\security_connector\\alts_security_connector.cc " + + "src\\core\\lib\\security\\security_connector\\load_system_roots_fallback.cc " + + "src\\core\\lib\\security\\security_connector\\load_system_roots_linux.cc " + "src\\core\\lib\\security\\security_connector\\local_security_connector.cc " + "src\\core\\lib\\security\\security_connector\\security_connector.cc " + "src\\core\\lib\\security\\transport\\client_auth_filter.cc " + @@ -355,11 +357,14 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\dns_resolver_ares.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver_posix.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_ev_driver_windows.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_fallback.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_posix.cc " + + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\c_ares\\grpc_ares_wrapper_windows.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\dns\\native\\dns_resolver.cc " + "src\\core\\ext\\filters\\client_channel\\resolver\\sockaddr\\sockaddr_resolver.cc " + - "src\\cpp\\ext\\filters\\census\\grpc_context.cc " + + "src\\core\\ext\\filters\\census\\grpc_context.cc " + "src\\core\\ext\\filters\\max_age\\max_age_filter.cc " + "src\\core\\ext\\filters\\message_size\\message_size_filter.cc " + "src\\core\\ext\\filters\\http\\client_authority_filter.cc " + @@ -660,6 +665,7 @@ if (PHP_GRPC != "no") { FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters"); + FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\census"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\ext\\filters\\client_channel\\lb_policy\\grpclb"); @@ -736,10 +742,6 @@ if (PHP_GRPC != "no") { FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\tsi\\alts\\zero_copy_frame_protector"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\tsi\\ssl"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\core\\tsi\\ssl\\session_cache"); - FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp"); - FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp\\ext"); - FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp\\ext\\filters"); - FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\cpp\\ext\\filters\\census"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\php"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\php\\ext"); FSO.CreateFolder(base_dir+"\\ext\\grpc\\src\\php\\ext\\grpc"); diff --git a/doc/command_line_tool.md b/doc/command_line_tool.md index 6337acaf60a..a373cbea627 100644 --- a/doc/command_line_tool.md +++ b/doc/command_line_tool.md @@ -41,12 +41,16 @@ repository, you need to run the following command to update submodules: git submodule update --init ``` -You also need to have the gflags library installed on your system. On Linux -systems, gflags can be installed with the following command: - +You also need to have the gflags library installed on your system. gflags can be +installed with the following command: +Linux: ``` sudo apt-get install libgflags-dev ``` +Mac systems with Homebrew: +``` +brew install gflags +``` Once the prerequisites are satisfied, you can build the command line tool with the command: diff --git a/doc/csharp/server_reflection.md b/doc/csharp/server_reflection.md new file mode 100644 index 00000000000..7827cbb21f0 --- /dev/null +++ b/doc/csharp/server_reflection.md @@ -0,0 +1,54 @@ +# gRPC C# Server Reflection + +This document shows how to use gRPC Server Reflection in gRPC C#. +Please see [C++ Server Reflection Tutorial](../server_reflection_tutorial.md) +for general information and more examples how to use server reflection. + +## Enable server reflection in C# servers + +C# Server Reflection is an add-on library. +To use it, first install the [Grpc.Reflection](https://www.nuget.org/packages/Grpc.Reflection/) +Nuget package into your project. + +Note that with C# you need to manually register the service +descriptors with the reflection service implementation when creating a server +(this isn't necessary with e.g. C++ or Java) +```csharp +// the reflection service will be aware of "Greeter" and "ServerReflection" services. +var reflectionServiceImpl = new ReflectionServiceImpl(Greeter.Descriptor, ServerReflection.Descriptor); +server = new Server() +{ + Services = + { + // the server will serve 2 services, the Greeter and the ServerReflection + ServerReflection.BindService(new GreeterImpl()), + ServerReflection.BindService(reflectionServiceImpl) + }, + Ports = { { "localhost", 50051, ServerCredentials.Insecure } } +}; +server.Start(); +``` + +After starting the server, you can verify that the server reflection +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): + + ```sh + $ grpc_cli ls localhost:50051 + ``` + + output: + ```sh + helloworld.Greeter + grpc.reflection.v1alpha.ServerReflection + ``` + + For more examples and instructions how to use the `grpc_cli` tool, + please refer to the [`grpc_cli` documentation](../command_line_tool.md) + and the [C++ Server Reflection Tutorial](../server_reflection_tutorial.md). + +## Additional Resources + +The [Server Reflection Protocol](../server-reflection.md) provides detailed +information about how the server reflection works and describes the server reflection +protocol in detail. diff --git a/doc/environment_variables.md b/doc/environment_variables.md index bf2de927a4f..b472f7e1263 100644 --- a/doc/environment_variables.md +++ b/doc/environment_variables.md @@ -135,3 +135,7 @@ some configuration as environment variables that can be set. if set, flow control will be effectively disabled. Max out all values and assume the remote peer does the same. Thus we can ignore any flow control bookkeeping, error checking, and decision making + +* grpc_cfstream + set to 1 to turn on CFStream experiment. With this experiment gRPC uses CFStream API to make TCP + connections. The option is only available on iOS platform and when macro GRPC_CFSTREAM is defined. diff --git a/doc/interop-test-descriptions.md b/doc/interop-test-descriptions.md index 9781925533f..3c33189196e 100644 --- a/doc/interop-test-descriptions.md +++ b/doc/interop-test-descriptions.md @@ -784,6 +784,32 @@ Client asserts: * received status message is the same as the sent message for both Procedure steps 1 and 2 +### special_status_message + +This test verifies Unicode and whitespace is correctly processed in status +message. "\t" is horizontal tab. "\r" is carriage return. "\n" is line feed. + +Server features: +* [UnaryCall][] +* [Echo Status][] + +Procedure: + 1. Client calls UnaryCall with: + + ``` + { + response_status:{ + code: 2 + message: "\t\ntest with whitespace\r\nand Unicode BMP ☺ and non-BMP 😈\t\n" + } + } + ``` + +Client asserts: +* received status code is the same as the sent code for Procedure step 1 +* received status message is the same as the sent message for Procedure step 1, + including all whitespace characters + ### unimplemented_method This test verifies that calling an unimplemented RPC method returns the diff --git a/doc/python/server_reflection.md b/doc/python/server_reflection.md new file mode 100644 index 00000000000..5a68e3c3019 --- /dev/null +++ b/doc/python/server_reflection.md @@ -0,0 +1,61 @@ +# gRPC Python Server Reflection + +This document shows how to use gRPC Server Reflection in gRPC Python. +Please see [C++ Server Reflection Tutorial](../server_reflection_tutorial.md) +for general information and more examples how to use server reflection. + +## Enable server reflection in Python servers + +gRPC Python Server Reflection is an add-on library. +To use it, first install the [grpcio-reflection](https://pypi.org/project/grpcio-reflection/) +PyPI package into your project. + +Note that with Python you need to manually register the service +descriptors with the reflection service implementation when creating a server +(this isn't necessary with e.g. C++ or Java) +```python +# add the following import statement to use server reflection +from grpc_reflection.v1alpha import reflection +# ... +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + # the reflection service will be aware of "Greeter" and "ServerReflection" services. + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() +``` + +Please see +[greeter_server_with_reflection.py](https://github.com/grpc/grpc/blob/master/examples/python/helloworld/greeter_server_with_reflection.py) +in the examples directory for the full example, which extends the gRPC [Python +`Greeter` example](https://github.com/grpc/tree/master/examples/python/helloworld) on a +reflection-enabled server. + +After starting the server, you can verify that the server reflection +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): + + ```sh + $ grpc_cli ls localhost:50051 + ``` + + output: + ```sh + grpc.reflection.v1alpha.ServerReflection + helloworld.Greeter + ``` + + For more examples and instructions how to use the `grpc_cli` tool, + please refer to the [`grpc_cli` documentation](../command_line_tool.md) + and the [C++ Server Reflection Tutorial](../server_reflection_tutorial.md). + +## Additional Resources + +The [Server Reflection Protocol](../server-reflection.md) provides detailed +information about how the server reflection works and describes the server reflection +protocol in detail. diff --git a/doc/server-reflection.md b/doc/server-reflection.md index cceee1647f6..3716dc5a21d 100644 --- a/doc/server-reflection.md +++ b/doc/server-reflection.md @@ -181,3 +181,16 @@ will need to index those FileDescriptorProtos by file and symbol and imports. One issue is that some grpc implementations are very loosely coupled with protobufs; in such implementations it probably makes sense to split apart these reflection APIs so as not to take an additional proto dependency. + +## Known Implementations + +Enabling server reflection differs language-to-language. Here are links to docs relevant to +each language: + +- [Java](https://github.com/grpc/grpc-java/blob/master/documentation/server-reflection-tutorial.md#enable-server-reflection) +- [Go](https://github.com/grpc/grpc-go/blob/master/Documentation/server-reflection-tutorial.md#enable-server-reflection) +- [C++](https://grpc.io/grpc/cpp/md_doc_server_reflection_tutorial.html) +- [C#](https://github.com/grpc/grpc/blob/master/doc/csharp/server_reflection.md) +- [Python](https://github.com/grpc/grpc/blob/master/doc/python/server_reflection.md) +- Ruby: not yet implemented [#2567](https://github.com/grpc/grpc/issues/2567) +- Node: not yet implemented [#2568](https://github.com/grpc/grpc/issues/2568) diff --git a/doc/server_reflection_tutorial.md b/doc/server_reflection_tutorial.md index ecb176723cc..06a257c1e87 100644 --- a/doc/server_reflection_tutorial.md +++ b/doc/server_reflection_tutorial.md @@ -10,7 +10,7 @@ RPCs. ### Enable server reflection in C++ servers -C++ Server Reflection is an add-on library, `libgrpc++_reflction`. To enable C++ +C++ Server Reflection is an add-on library, `libgrpc++_reflection`. To enable C++ server reflection, you can link this library to your server binary. Some platforms (e.g. Ubuntu 11.10 onwards) only link in libraries that directly diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md new file mode 100644 index 00000000000..176c8d8f247 --- /dev/null +++ b/doc/ssl-performance.md @@ -0,0 +1,36 @@ +# SSL in gRPC and performance + +The SSL requirement of gRPC isn't necessarily making it easy to integrate. The HTTP/2 protocol requires ALPN support, which is a fairly new handshake protocol only supported by recent implementations. + +As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code +(by setting the `OPENSSL_NO_ASM` option), which can impact performance by an order of magnitude when processing encrypted streams. + +## gRPC C++: Building from Source + +Build system | Condition | Platform | Uses assembly optimizations +---|---|---|-- +Makefile | with OpenSSL 1.0.2 development files | all | :heavy_check_mark: +Makefile | all other cases | all | :x: +Bazel | | Linux | :heavy_check_mark: +Bazel | | MacOS | :heavy_check_mark: +Bazel | | Windows | :x: +CMake | boringssl from submodule (default) | all | :x: +CMake | pre-installed OpenSSL 1.0.2+ (`gRPC_SSL_PROVIDER=package`) | all | :heavy_check_mark: + +## Other Languages: Binary/Source Packages + +In addition, we are shipping packages for language implementations. These packages are source packages, but also have pre-built binaries being distributed. Building packages from source may give a different result in some cases. + +Language | From source | Platform | Uses assembly optimizations +---|---|---|--- +C# | n/a | all | :x: +Node.JS | n/a | Linux | :heavy_check_mark: +Node.JS | n/a | MacOS | :heavy_check_mark: +Node.JS | n/a | Windows | :x: +Electron | n/a | all | :heavy_check_mark: +ObjC | Yes | iOS | :x: +PHP | Yes | all | Same as the `Makefile` case from above +PHP | No | all | :x: +Python | n/a | all | :x: +Ruby | No | all | :x: + diff --git a/doc/statuscodes.md b/doc/statuscodes.md index f2df9e00dec..06fbe5c8fe8 100644 --- a/doc/statuscodes.md +++ b/doc/statuscodes.md @@ -38,6 +38,7 @@ situations in which they are generated. | Error parsing response proto | INTERNAL | Client| | Error parsing request proto | INTERNAL | Server| | Sent or received message was larger than configured limit | RESOURCE_EXHAUSTED | Both | +| Keepalive watchdog times out | INTERNAL | Both | The following status codes are never generated by the library: - INVALID_ARGUMENT @@ -47,3 +48,5 @@ The following status codes are never generated by the library: - ABORTED - OUT_OF_RANGE - DATA_LOSS + +Applications that may wish to [retry](https://github.com/grpc/proposal/blob/master/A6-client-retries.md) failed RPCs must decide which status codes on which to retry. As shown in the table above, the gRPC library can generate the same status code for different cases. Server applications can also return those same status codes. Therefore, there is no fixed list of status codes on which it is appropriate to retry in all applications. As a result, individual applications must make their own determination as to which status codes should cause an RPC to be retried. diff --git a/etc/roots.pem b/etc/roots.pem index 5dbd1ae6edb..c22dfe69f87 100644 --- a/etc/roots.pem +++ b/etc/roots.pem @@ -3734,169 +3734,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4440,3 +4277,43 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- diff --git a/examples/csharp/helloworld-from-cli/Greeter.sln b/examples/csharp/Helloworld/Greeter.sln similarity index 100% rename from examples/csharp/helloworld-from-cli/Greeter.sln rename to examples/csharp/Helloworld/Greeter.sln diff --git a/examples/csharp/helloworld-from-cli/Greeter/Greeter.csproj b/examples/csharp/Helloworld/Greeter/Greeter.csproj similarity index 79% rename from examples/csharp/helloworld-from-cli/Greeter/Greeter.csproj rename to examples/csharp/Helloworld/Greeter/Greeter.csproj index 3bff4a576bb..3d4be5da6b4 100644 --- a/examples/csharp/helloworld-from-cli/Greeter/Greeter.csproj +++ b/examples/csharp/Helloworld/Greeter/Greeter.csproj @@ -11,8 +11,8 @@ - - + + diff --git a/examples/csharp/helloworld-from-cli/Greeter/Helloworld.cs b/examples/csharp/Helloworld/Greeter/Helloworld.cs similarity index 100% rename from examples/csharp/helloworld-from-cli/Greeter/Helloworld.cs rename to examples/csharp/Helloworld/Greeter/Helloworld.cs diff --git a/examples/csharp/helloworld-from-cli/Greeter/HelloworldGrpc.cs b/examples/csharp/Helloworld/Greeter/HelloworldGrpc.cs similarity index 100% rename from examples/csharp/helloworld-from-cli/Greeter/HelloworldGrpc.cs rename to examples/csharp/Helloworld/Greeter/HelloworldGrpc.cs diff --git a/examples/csharp/helloworld-from-cli/GreeterClient/GreeterClient.csproj b/examples/csharp/Helloworld/GreeterClient/GreeterClient.csproj similarity index 100% rename from examples/csharp/helloworld-from-cli/GreeterClient/GreeterClient.csproj rename to examples/csharp/Helloworld/GreeterClient/GreeterClient.csproj diff --git a/examples/csharp/helloworld-from-cli/GreeterClient/Program.cs b/examples/csharp/Helloworld/GreeterClient/Program.cs similarity index 100% rename from examples/csharp/helloworld-from-cli/GreeterClient/Program.cs rename to examples/csharp/Helloworld/GreeterClient/Program.cs diff --git a/examples/csharp/helloworld-from-cli/GreeterServer/GreeterServer.csproj b/examples/csharp/Helloworld/GreeterServer/GreeterServer.csproj similarity index 100% rename from examples/csharp/helloworld-from-cli/GreeterServer/GreeterServer.csproj rename to examples/csharp/Helloworld/GreeterServer/GreeterServer.csproj diff --git a/examples/csharp/helloworld-from-cli/GreeterServer/Program.cs b/examples/csharp/Helloworld/GreeterServer/Program.cs similarity index 100% rename from examples/csharp/helloworld-from-cli/GreeterServer/Program.cs rename to examples/csharp/Helloworld/GreeterServer/Program.cs diff --git a/examples/csharp/helloworld-from-cli/README.md b/examples/csharp/Helloworld/README.md similarity index 73% rename from examples/csharp/helloworld-from-cli/README.md rename to examples/csharp/Helloworld/README.md index b780fa1b2fd..e6031794388 100644 --- a/examples/csharp/helloworld-from-cli/README.md +++ b/examples/csharp/Helloworld/README.md @@ -3,9 +3,6 @@ gRPC in 3 minutes (C#) BACKGROUND ------------- -This is a different version of the helloworld example, using the dotnet sdk -tools to build and run. - For this sample, we've already generated the server and client stubs from [helloworld.proto][]. Example projects in this directory depend on the [Grpc](https://www.nuget.org/packages/Grpc/) @@ -15,17 +12,19 @@ which have been already added to the project for you. PREREQUISITES ------------- -- The [.NET Core SDK](https://www.microsoft.com/net/core). +- The [.NET Core SDK](https://www.microsoft.com/net/core) (version 2+ is recommended) + +You can also build the example directly using Visual Studio 2017, but it's not a requirement. BUILD ------- -From the `examples/csharp/helloworld-from-cli` directory: - -- `dotnet restore Greeter.sln` +From the `examples/csharp/Helloworld` directory: - `dotnet build Greeter.sln` +(if you're using dotnet SDK 1.x you need to run `dotnet restore Greeter.sln` first) + Try it! ------- diff --git a/examples/csharp/helloworld-from-cli/generate_protos.bat b/examples/csharp/Helloworld/generate_protos.bat similarity index 100% rename from examples/csharp/helloworld-from-cli/generate_protos.bat rename to examples/csharp/Helloworld/generate_protos.bat diff --git a/examples/csharp/helloworld/Greeter.sln b/examples/csharp/HelloworldLegacyCsproj/Greeter.sln similarity index 100% rename from examples/csharp/helloworld/Greeter.sln rename to examples/csharp/HelloworldLegacyCsproj/Greeter.sln diff --git a/examples/csharp/helloworld/Greeter/.gitignore b/examples/csharp/HelloworldLegacyCsproj/Greeter/.gitignore similarity index 100% rename from examples/csharp/helloworld/Greeter/.gitignore rename to examples/csharp/HelloworldLegacyCsproj/Greeter/.gitignore diff --git a/examples/csharp/helloworld/Greeter/Greeter.csproj b/examples/csharp/HelloworldLegacyCsproj/Greeter/Greeter.csproj similarity index 72% rename from examples/csharp/helloworld/Greeter/Greeter.csproj rename to examples/csharp/HelloworldLegacyCsproj/Greeter/Greeter.csproj index d2597f13b5f..ab584c86d3e 100644 --- a/examples/csharp/helloworld/Greeter/Greeter.csproj +++ b/examples/csharp/HelloworldLegacyCsproj/Greeter/Greeter.csproj @@ -36,15 +36,15 @@ ..\packages\Google.Protobuf.3.5.0\lib\net45\Google.Protobuf.dll True - - ..\packages\Grpc.Core.1.8.0\lib\net45\Grpc.Core.dll - True - ..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll True + + ..\packages\Grpc.Core.1.13.1\lib\net45\Grpc.Core.dll + + @@ -62,11 +62,5 @@ - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - + \ No newline at end of file diff --git a/examples/csharp/helloworld/Greeter/Helloworld.cs b/examples/csharp/HelloworldLegacyCsproj/Greeter/Helloworld.cs similarity index 100% rename from examples/csharp/helloworld/Greeter/Helloworld.cs rename to examples/csharp/HelloworldLegacyCsproj/Greeter/Helloworld.cs diff --git a/examples/csharp/helloworld/Greeter/HelloworldGrpc.cs b/examples/csharp/HelloworldLegacyCsproj/Greeter/HelloworldGrpc.cs similarity index 100% rename from examples/csharp/helloworld/Greeter/HelloworldGrpc.cs rename to examples/csharp/HelloworldLegacyCsproj/Greeter/HelloworldGrpc.cs diff --git a/examples/csharp/helloworld/Greeter/Properties/AssemblyInfo.cs b/examples/csharp/HelloworldLegacyCsproj/Greeter/Properties/AssemblyInfo.cs similarity index 100% rename from examples/csharp/helloworld/Greeter/Properties/AssemblyInfo.cs rename to examples/csharp/HelloworldLegacyCsproj/Greeter/Properties/AssemblyInfo.cs diff --git a/examples/csharp/helloworld/Greeter/packages.config b/examples/csharp/HelloworldLegacyCsproj/Greeter/packages.config similarity index 50% rename from examples/csharp/helloworld/Greeter/packages.config rename to examples/csharp/HelloworldLegacyCsproj/Greeter/packages.config index 38297f8d617..8e61429a8ea 100644 --- a/examples/csharp/helloworld/Greeter/packages.config +++ b/examples/csharp/HelloworldLegacyCsproj/Greeter/packages.config @@ -1,8 +1,8 @@  - - - + + + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterClient/.gitignore b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/.gitignore similarity index 100% rename from examples/csharp/helloworld/GreeterClient/.gitignore rename to examples/csharp/HelloworldLegacyCsproj/GreeterClient/.gitignore diff --git a/examples/csharp/helloworld/GreeterClient/GreeterClient.csproj b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/GreeterClient.csproj similarity index 72% rename from examples/csharp/helloworld/GreeterClient/GreeterClient.csproj rename to examples/csharp/HelloworldLegacyCsproj/GreeterClient/GreeterClient.csproj index 470749a2b2b..2d2961d1289 100644 --- a/examples/csharp/helloworld/GreeterClient/GreeterClient.csproj +++ b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/GreeterClient.csproj @@ -36,15 +36,15 @@ ..\packages\Google.Protobuf.3.5.0\lib\net45\Google.Protobuf.dll True - - ..\packages\Grpc.Core.1.8.0\lib\net45\Grpc.Core.dll - True - ..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll True + + ..\packages\Grpc.Core.1.13.1\lib\net45\Grpc.Core.dll + + @@ -60,11 +60,5 @@ - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterClient/Program.cs b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/Program.cs similarity index 100% rename from examples/csharp/helloworld/GreeterClient/Program.cs rename to examples/csharp/HelloworldLegacyCsproj/GreeterClient/Program.cs diff --git a/examples/csharp/helloworld/GreeterClient/Properties/AssemblyInfo.cs b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/Properties/AssemblyInfo.cs similarity index 100% rename from examples/csharp/helloworld/GreeterClient/Properties/AssemblyInfo.cs rename to examples/csharp/HelloworldLegacyCsproj/GreeterClient/Properties/AssemblyInfo.cs diff --git a/examples/csharp/helloworld/GreeterClient/packages.config b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/packages.config similarity index 59% rename from examples/csharp/helloworld/GreeterClient/packages.config rename to examples/csharp/HelloworldLegacyCsproj/GreeterClient/packages.config index 4b3684edfd3..da7dbcd8cb5 100644 --- a/examples/csharp/helloworld/GreeterClient/packages.config +++ b/examples/csharp/HelloworldLegacyCsproj/GreeterClient/packages.config @@ -1,7 +1,7 @@  - - + + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterServer/.gitignore b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/.gitignore similarity index 100% rename from examples/csharp/helloworld/GreeterServer/.gitignore rename to examples/csharp/HelloworldLegacyCsproj/GreeterServer/.gitignore diff --git a/examples/csharp/helloworld/GreeterServer/GreeterServer.csproj b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/GreeterServer.csproj similarity index 72% rename from examples/csharp/helloworld/GreeterServer/GreeterServer.csproj rename to examples/csharp/HelloworldLegacyCsproj/GreeterServer/GreeterServer.csproj index 82e2961cad2..1d47d705955 100644 --- a/examples/csharp/helloworld/GreeterServer/GreeterServer.csproj +++ b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/GreeterServer.csproj @@ -36,15 +36,15 @@ ..\packages\Google.Protobuf.3.5.0\lib\net45\Google.Protobuf.dll True - - ..\packages\Grpc.Core.1.8.0\lib\net45\Grpc.Core.dll - True - ..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll True + + ..\packages\Grpc.Core.1.13.1\lib\net45\Grpc.Core.dll + + @@ -60,11 +60,5 @@ - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - + \ No newline at end of file diff --git a/examples/csharp/helloworld/GreeterServer/Program.cs b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/Program.cs similarity index 100% rename from examples/csharp/helloworld/GreeterServer/Program.cs rename to examples/csharp/HelloworldLegacyCsproj/GreeterServer/Program.cs diff --git a/examples/csharp/helloworld/GreeterServer/Properties/AssemblyInfo.cs b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/Properties/AssemblyInfo.cs similarity index 100% rename from examples/csharp/helloworld/GreeterServer/Properties/AssemblyInfo.cs rename to examples/csharp/HelloworldLegacyCsproj/GreeterServer/Properties/AssemblyInfo.cs diff --git a/examples/csharp/helloworld/GreeterServer/packages.config b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/packages.config similarity index 59% rename from examples/csharp/helloworld/GreeterServer/packages.config rename to examples/csharp/HelloworldLegacyCsproj/GreeterServer/packages.config index 4b3684edfd3..da7dbcd8cb5 100644 --- a/examples/csharp/helloworld/GreeterServer/packages.config +++ b/examples/csharp/HelloworldLegacyCsproj/GreeterServer/packages.config @@ -1,7 +1,7 @@  - - + + \ No newline at end of file diff --git a/examples/csharp/helloworld/README.md b/examples/csharp/HelloworldLegacyCsproj/README.md similarity index 82% rename from examples/csharp/helloworld/README.md rename to examples/csharp/HelloworldLegacyCsproj/README.md index 55e3ab70305..6d42c5ef258 100644 --- a/examples/csharp/helloworld/README.md +++ b/examples/csharp/HelloworldLegacyCsproj/README.md @@ -3,6 +3,11 @@ gRPC in 3 minutes (C#) BACKGROUND ------------- +This is a different version of the helloworld example, using the old-style .csproj +files supported by VS2013 and VS2015 (and older versions of mono). +You can still use gRPC with the old-style .csproj files, but [using the new-style +.csproj projects](../helloworld/README.md) (supported by VS2017 and dotnet SDK) is recommended. + For this sample, we've already generated the server and client stubs from [helloworld.proto][]. Example projects depend on the [Grpc](https://www.nuget.org/packages/Grpc/), [Grpc.Tools](https://www.nuget.org/packages/Grpc.Tools/) @@ -28,7 +33,7 @@ BUILD # Using Monodevelop or Xamarin Studio The nuget add-in available for Xamarin Studio and Monodevelop IDEs is too old to -download all of the nuget dependencies of gRPC. One alternative to is to use the dotnet command line tools instead (see [helloworld-from-cli]). +download all of the nuget dependencies of gRPC. Using these IDEs, a workaround is as follows: * Obtain a nuget executable for your platform and update it with @@ -62,6 +67,5 @@ Tutorial You can find a more detailed tutorial in [gRPC Basics: C#][] -[helloworld-from-cli]:../helloworld-from-cli/README.md [helloworld.proto]:../../protos/helloworld.proto [gRPC Basics: C#]:https://grpc.io/docs/tutorials/basic/csharp.html diff --git a/examples/csharp/helloworld/generate_protos.bat b/examples/csharp/HelloworldLegacyCsproj/generate_protos.bat similarity index 100% rename from examples/csharp/helloworld/generate_protos.bat rename to examples/csharp/HelloworldLegacyCsproj/generate_protos.bat diff --git a/examples/csharp/HelloworldXamarin/.gitignore b/examples/csharp/HelloworldXamarin/.gitignore new file mode 100644 index 00000000000..bf793edcc66 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/.gitignore @@ -0,0 +1,41 @@ +# Autosave files +*~ + +# build +[Oo]bj/ +[Bb]in/ +packages/ +TestResults/ + +# globs +Makefile.in +*.DS_Store +*.sln.cache +*.suo +*.cache +*.pidb +*.userprefs +*.usertasks +config.log +config.make +config.status +aclocal.m4 +install-sh +autom4te.cache/ +*.user +*.tar.gz +tarballs/ +test-results/ +Thumbs.db +.vs/ + +# Mac bundle stuff +*.dmg +*.app + +# resharper +*_Resharper.* +*.Resharper + +# dotCover +*.dotCover diff --git a/examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt b/examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt new file mode 100644 index 00000000000..a9b0638eb1b --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/Assets/AboutAssets.txt @@ -0,0 +1,19 @@ +Any raw assets you want to be deployed with your application can be placed in +this directory (and child directories) and given a Build Action of "AndroidAsset". + +These files will be deployed with your package and will be accessible using Android's +AssetManager, like this: + +public class ReadAsset : Activity +{ + protected override void OnCreate (Bundle bundle) + { + base.OnCreate (bundle); + + InputStream input = Assets.Open ("my_asset.txt"); + } +} + +Additionally, some Android functions will automatically load asset files: + +Typeface tf = Typeface.CreateFromAsset (Context.Assets, "fonts/samplefont.ttf"); diff --git a/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj new file mode 100644 index 00000000000..b5ca8490a48 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/HelloworldXamarin.Droid.csproj @@ -0,0 +1,83 @@ + + + + Debug + AnyCPU + {B9B0D41C-1C07-4590-A919-5865E741B2EA} + {EFBA0AD7-5A72-4C68-AF49-83D382785DCF};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + Library + HelloworldXamarin.Droid + HelloworldXamarin.Droid + v8.1 + True + Resources\Resource.designer.cs + Resource + Properties\AndroidManifest.xml + Resources + Assets + true + + + true + full + false + bin\Debug + DEBUG; + prompt + 4 + None + + + true + pdbonly + true + bin\Release + prompt + 4 + true + false + + + + + + + + + + ..\packages\System.Runtime.Loader.4.0.0\lib\netstandard1.5\System.Runtime.Loader.dll + + + ..\packages\System.Interactive.Async.3.1.1\lib\netstandard1.3\System.Interactive.Async.dll + + + ..\packages\Grpc.Core.1.15.0-dev\lib\netstandard1.5\Grpc.Core.dll + + + ..\packages\Google.Protobuf.3.6.0\lib\netstandard1.0\Google.Protobuf.dll + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs b/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs new file mode 100644 index 00000000000..353943ced66 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/Droid/MainActivity.cs @@ -0,0 +1,84 @@ +#region Copyright notice and license + +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using Android.App; +using Android.Widget; +using Android.OS; +using System.Threading.Tasks; +using Grpc.Core; +using Helloworld; + +namespace HelloworldXamarin.Droid +{ + [Activity(Label = "HelloworldXamarin", MainLauncher = true, Icon = "@mipmap/icon")] + public class MainActivity : Activity + { + const int Port = 50051; + int count = 1; + + protected override void OnCreate(Bundle savedInstanceState) + { + base.OnCreate(savedInstanceState); + + // Set our view from the "main" layout resource + SetContentView(Resource.Layout.Main); + + // Get our button from the layout resource, + // and attach an event to it + Button button = FindViewById + + + + + + + + + + + + + + + + diff --git a/examples/csharp/HelloworldXamarin/iOS/ViewController.cs b/examples/csharp/HelloworldXamarin/iOS/ViewController.cs new file mode 100644 index 00000000000..c553c1eb483 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/iOS/ViewController.cs @@ -0,0 +1,91 @@ +#region Copyright notice and license + +// Copyright 2018 The gRPC Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#endregion + +using System; +using System.Threading.Tasks; + +using Grpc.Core; +using Helloworld; + +using UIKit; + +namespace HelloworldXamarin.iOS +{ + public partial class ViewController : UIViewController + { + const int Port = 50051; + int count = 1; + + public ViewController(IntPtr handle) : base(handle) + { + } + + public override void ViewDidLoad() + { + base.ViewDidLoad(); + + // Perform any additional setup after loading the view, typically from a nib. + Button.AccessibilityIdentifier = "myButton"; + Button.TouchUpInside += delegate + { + var title = SayHello(); + Button.SetTitle(title, UIControlState.Normal); + }; + } + + public override void DidReceiveMemoryWarning() + { + base.DidReceiveMemoryWarning(); + // Release any cached data, images, etc that aren't in use. + } + + private string SayHello() + { + Server server = new Server + { + Services = { Greeter.BindService(new GreeterImpl()) }, + Ports = { new ServerPort("localhost", Port, ServerCredentials.Insecure) } + }; + server.Start(); + + Channel channel = new Channel("localhost:50051", ChannelCredentials.Insecure); + + var client = new Greeter.GreeterClient(channel); + string user = "Xamarin " + count; + + var reply = client.SayHello(new HelloRequest { Name = user }); + + channel.ShutdownAsync().Wait(); + server.ShutdownAsync().Wait(); + + count++; + + return "Greeting: " + reply.Message; + } + + + class GreeterImpl : Greeter.GreeterBase + { + // Server side handler of the SayHello RPC + public override Task SayHello(HelloRequest request, ServerCallContext context) + { + return Task.FromResult(new HelloReply { Message = "Hello " + request.Name }); + } + } + } +} diff --git a/examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs b/examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs new file mode 100644 index 00000000000..2677a068dca --- /dev/null +++ b/examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs @@ -0,0 +1,25 @@ +// +// This file has been generated automatically by MonoDevelop to store outlets and +// actions made in the Xcode designer. If it is removed, they will be lost. +// Manual changes to this file may not be handled correctly. +// +using Foundation; + +namespace HelloworldXamarin.iOS +{ + [Register("ViewController")] + partial class ViewController + { + [Outlet] + UIKit.UIButton Button { get; set; } + + void ReleaseDesignerOutlets() + { + if (Button != null) + { + Button.Dispose(); + Button = null; + } + } + } +} diff --git a/examples/csharp/HelloworldXamarin/iOS/packages.config b/examples/csharp/HelloworldXamarin/iOS/packages.config new file mode 100644 index 00000000000..ce4bceb62a8 --- /dev/null +++ b/examples/csharp/HelloworldXamarin/iOS/packages.config @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/examples/csharp/route_guide/.gitignore b/examples/csharp/RouteGuide/.gitignore similarity index 100% rename from examples/csharp/route_guide/.gitignore rename to examples/csharp/RouteGuide/.gitignore diff --git a/examples/csharp/route_guide/README.md b/examples/csharp/RouteGuide/README.md similarity index 100% rename from examples/csharp/route_guide/README.md rename to examples/csharp/RouteGuide/README.md diff --git a/examples/csharp/route_guide/RouteGuide.sln b/examples/csharp/RouteGuide/RouteGuide.sln similarity index 87% rename from examples/csharp/route_guide/RouteGuide.sln rename to examples/csharp/RouteGuide/RouteGuide.sln index 00065b0ba9d..73e6e306b16 100644 --- a/examples/csharp/route_guide/RouteGuide.sln +++ b/examples/csharp/RouteGuide/RouteGuide.sln @@ -1,13 +1,13 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.31101.0 +# Visual Studio 15 +VisualStudioVersion = 15.0.26228.4 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RouteGuide", "RouteGuide\RouteGuide.csproj", "{49954D9C-5F17-4662-96B2-73BE833DD81A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RouteGuide", "RouteGuide\RouteGuide.csproj", "{49954D9C-5F17-4662-96B2-73BE833DD81A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RouteGuideClient", "RouteGuideClient\RouteGuideClient.csproj", "{D47BE663-4DE3-4206-B7A8-EA3FA066DADC}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RouteGuideClient", "RouteGuideClient\RouteGuideClient.csproj", "{D47BE663-4DE3-4206-B7A8-EA3FA066DADC}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RouteGuideServer", "RouteGuideServer\RouteGuideServer.csproj", "{4B7C7794-BE24-4477-ACE7-18259EB73D27}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "RouteGuideServer", "RouteGuideServer\RouteGuideServer.csproj", "{4B7C7794-BE24-4477-ACE7-18259EB73D27}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/examples/csharp/route_guide/RouteGuide/RouteGuide.cs b/examples/csharp/RouteGuide/RouteGuide/RouteGuide.cs similarity index 100% rename from examples/csharp/route_guide/RouteGuide/RouteGuide.cs rename to examples/csharp/RouteGuide/RouteGuide/RouteGuide.cs diff --git a/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj b/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj new file mode 100644 index 00000000000..7419f1a277d --- /dev/null +++ b/examples/csharp/RouteGuide/RouteGuide/RouteGuide.csproj @@ -0,0 +1,25 @@ + + + + RouteGuide + netcoreapp1.0 + portable + RouteGuide + RouteGuide + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/examples/csharp/route_guide/RouteGuide/RouteGuideGrpc.cs b/examples/csharp/RouteGuide/RouteGuide/RouteGuideGrpc.cs similarity index 100% rename from examples/csharp/route_guide/RouteGuide/RouteGuideGrpc.cs rename to examples/csharp/RouteGuide/RouteGuide/RouteGuideGrpc.cs diff --git a/examples/csharp/route_guide/RouteGuide/RouteGuideUtil.cs b/examples/csharp/RouteGuide/RouteGuide/RouteGuideUtil.cs similarity index 100% rename from examples/csharp/route_guide/RouteGuide/RouteGuideUtil.cs rename to examples/csharp/RouteGuide/RouteGuide/RouteGuideUtil.cs diff --git a/examples/csharp/route_guide/RouteGuide/route_guide_db.json b/examples/csharp/RouteGuide/RouteGuide/route_guide_db.json similarity index 100% rename from examples/csharp/route_guide/RouteGuide/route_guide_db.json rename to examples/csharp/RouteGuide/RouteGuide/route_guide_db.json diff --git a/examples/csharp/route_guide/RouteGuideClient/Program.cs b/examples/csharp/RouteGuide/RouteGuideClient/Program.cs similarity index 100% rename from examples/csharp/route_guide/RouteGuideClient/Program.cs rename to examples/csharp/RouteGuide/RouteGuideClient/Program.cs diff --git a/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj b/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj new file mode 100644 index 00000000000..96cc204ba37 --- /dev/null +++ b/examples/csharp/RouteGuide/RouteGuideClient/RouteGuideClient.csproj @@ -0,0 +1,16 @@ + + + + RouteGuideClient + netcoreapp1.0 + portable + RouteGuideClient + Exe + RouteGuideClient + + + + + + + diff --git a/examples/csharp/route_guide/RouteGuideServer/Program.cs b/examples/csharp/RouteGuide/RouteGuideServer/Program.cs similarity index 100% rename from examples/csharp/route_guide/RouteGuideServer/Program.cs rename to examples/csharp/RouteGuide/RouteGuideServer/Program.cs diff --git a/examples/csharp/route_guide/RouteGuideServer/RouteGuideImpl.cs b/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideImpl.cs similarity index 100% rename from examples/csharp/route_guide/RouteGuideServer/RouteGuideImpl.cs rename to examples/csharp/RouteGuide/RouteGuideServer/RouteGuideImpl.cs diff --git a/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj b/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj new file mode 100644 index 00000000000..aa6315bf818 --- /dev/null +++ b/examples/csharp/RouteGuide/RouteGuideServer/RouteGuideServer.csproj @@ -0,0 +1,16 @@ + + + + RouteGuideServer + netcoreapp1.0 + portable + RouteGuideServer + Exe + RouteGuideServer + + + + + + + diff --git a/examples/csharp/route_guide/generate_protos.bat b/examples/csharp/RouteGuide/generate_protos.bat similarity index 100% rename from examples/csharp/route_guide/generate_protos.bat rename to examples/csharp/RouteGuide/generate_protos.bat diff --git a/examples/csharp/route_guide/RouteGuide/Properties/AssemblyInfo.cs b/examples/csharp/route_guide/RouteGuide/Properties/AssemblyInfo.cs deleted file mode 100644 index dfee25c14cf..00000000000 --- a/examples/csharp/route_guide/RouteGuide/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,54 +0,0 @@ -#region Copyright notice and license - -// Copyright 2015 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#endregion - -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("RouteGuide")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("RouteGuide")] -[assembly: AssemblyCopyright("Copyright © 2015")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("ef6b85bc-ac27-46de-8714-a658236cc6fb")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj b/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj deleted file mode 100644 index e66e986f71f..00000000000 --- a/examples/csharp/route_guide/RouteGuide/RouteGuide.csproj +++ /dev/null @@ -1,92 +0,0 @@ - - - - - Debug - AnyCPU - {49954D9C-5F17-4662-96B2-73BE833DD81A} - Library - Properties - RouteGuide - RouteGuide - v4.5 - 512 - - - - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Google.Protobuf.3.5.0\lib\net45\Google.Protobuf.dll - True - - - ..\packages\Grpc.Core.1.8.0\lib\net45\Grpc.Core.dll - True - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - - - - - ..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll - True - - - - - - - - - - - - - - - - protos\route_guide.proto - - - generate_protos.bat - - - - PreserveNewest - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - \ No newline at end of file diff --git a/examples/csharp/route_guide/RouteGuide/packages.config b/examples/csharp/route_guide/RouteGuide/packages.config deleted file mode 100644 index fe2c995f262..00000000000 --- a/examples/csharp/route_guide/RouteGuide/packages.config +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/examples/csharp/route_guide/RouteGuideClient/Properties/AssemblyInfo.cs b/examples/csharp/route_guide/RouteGuideClient/Properties/AssemblyInfo.cs deleted file mode 100644 index 4ccdf701d3f..00000000000 --- a/examples/csharp/route_guide/RouteGuideClient/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,54 +0,0 @@ -#region Copyright notice and license - -// Copyright 2015 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#endregion - -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("RouteGuideClient")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("RouteGuideClient")] -[assembly: AssemblyCopyright("Copyright © 2015")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("914644eb-47cd-4a37-9fba-5e62dd432333")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/csharp/route_guide/RouteGuideClient/RouteGuideClient.csproj b/examples/csharp/route_guide/RouteGuideClient/RouteGuideClient.csproj deleted file mode 100644 index 612f60cba58..00000000000 --- a/examples/csharp/route_guide/RouteGuideClient/RouteGuideClient.csproj +++ /dev/null @@ -1,89 +0,0 @@ - - - - - Debug - AnyCPU - {D47BE663-4DE3-4206-B7A8-EA3FA066DADC} - Exe - Properties - RouteGuideClient - RouteGuideClient - v4.5 - 512 - - - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Google.Protobuf.3.5.0\lib\net45\Google.Protobuf.dll - True - - - ..\packages\Grpc.Core.1.8.0\lib\net45\Grpc.Core.dll - True - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - - - - - ..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll - True - - - - - - - - - - - - - - - - - {49954d9c-5f17-4662-96b2-73be833dd81a} - RouteGuide - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - \ No newline at end of file diff --git a/examples/csharp/route_guide/RouteGuideClient/packages.config b/examples/csharp/route_guide/RouteGuideClient/packages.config deleted file mode 100644 index fe2c995f262..00000000000 --- a/examples/csharp/route_guide/RouteGuideClient/packages.config +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/examples/csharp/route_guide/RouteGuideServer/Properties/AssemblyInfo.cs b/examples/csharp/route_guide/RouteGuideServer/Properties/AssemblyInfo.cs deleted file mode 100644 index 679bc4c9131..00000000000 --- a/examples/csharp/route_guide/RouteGuideServer/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,54 +0,0 @@ -#region Copyright notice and license - -// Copyright 2015 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#endregion - -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("RouteGuideServer")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("RouteGuideServer")] -[assembly: AssemblyCopyright("Copyright © 2015")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("908bdeef-05cc-42bf-9498-c4c573df8925")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.0")] -[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/examples/csharp/route_guide/RouteGuideServer/RouteGuideServer.csproj b/examples/csharp/route_guide/RouteGuideServer/RouteGuideServer.csproj deleted file mode 100644 index 4d9d9d74f2c..00000000000 --- a/examples/csharp/route_guide/RouteGuideServer/RouteGuideServer.csproj +++ /dev/null @@ -1,90 +0,0 @@ - - - - - Debug - AnyCPU - {4B7C7794-BE24-4477-ACE7-18259EB73D27} - Exe - Properties - RouteGuideServer - RouteGuideServer - v4.5 - 512 - - - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Google.Protobuf.3.5.0\lib\net45\Google.Protobuf.dll - True - - - ..\packages\Grpc.Core.1.8.0\lib\net45\Grpc.Core.dll - True - - - False - ..\packages\Newtonsoft.Json.7.0.1\lib\net45\Newtonsoft.Json.dll - - - - - ..\packages\System.Interactive.Async.3.1.1\lib\net45\System.Interactive.Async.dll - True - - - - - - - - - - - - - - - - - - {49954d9c-5f17-4662-96b2-73be833dd81a} - RouteGuide - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - \ No newline at end of file diff --git a/examples/csharp/route_guide/RouteGuideServer/packages.config b/examples/csharp/route_guide/RouteGuideServer/packages.config deleted file mode 100644 index 2bb1f0d0bf0..00000000000 --- a/examples/csharp/route_guide/RouteGuideServer/packages.config +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/examples/node/dynamic_codegen/greeter_client.js b/examples/node/dynamic_codegen/greeter_client.js index 9fd1f88f4fb..c289f501ee1 100644 --- a/examples/node/dynamic_codegen/greeter_client.js +++ b/examples/node/dynamic_codegen/greeter_client.js @@ -19,7 +19,16 @@ var PROTO_PATH = __dirname + '/../../protos/helloworld.proto'; var grpc = require('grpc'); -var hello_proto = grpc.load(PROTO_PATH).helloworld; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var hello_proto = grpc.loadPackageDefinition(packageDefinition).helloworld; function main() { var client = new hello_proto.Greeter('localhost:50051', diff --git a/examples/node/dynamic_codegen/greeter_server.js b/examples/node/dynamic_codegen/greeter_server.js index 180f96c28b8..023257ecf08 100644 --- a/examples/node/dynamic_codegen/greeter_server.js +++ b/examples/node/dynamic_codegen/greeter_server.js @@ -19,7 +19,16 @@ var PROTO_PATH = __dirname + '/../../protos/helloworld.proto'; var grpc = require('grpc'); -var hello_proto = grpc.load(PROTO_PATH).helloworld; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var hello_proto = grpc.loadPackageDefinition(packageDefinition).helloworld; /** * Implements the SayHello RPC method. diff --git a/examples/node/dynamic_codegen/route_guide/route_guide_client.js b/examples/node/dynamic_codegen/route_guide/route_guide_client.js index 703cfd29028..34a44bd6a9e 100644 --- a/examples/node/dynamic_codegen/route_guide/route_guide_client.js +++ b/examples/node/dynamic_codegen/route_guide/route_guide_client.js @@ -24,7 +24,16 @@ var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(PROTO_PATH).routeguide; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var routeguide = grpc.loadPackageDefinition(packageDefinition).routeguide; var client = new routeguide.RouteGuide('localhost:50051', grpc.credentials.createInsecure()); diff --git a/examples/node/dynamic_codegen/route_guide/route_guide_server.js b/examples/node/dynamic_codegen/route_guide/route_guide_server.js index 3819c092eb5..106491c1a10 100644 --- a/examples/node/dynamic_codegen/route_guide/route_guide_server.js +++ b/examples/node/dynamic_codegen/route_guide/route_guide_server.js @@ -23,7 +23,16 @@ var parseArgs = require('minimist'); var path = require('path'); var _ = require('lodash'); var grpc = require('grpc'); -var routeguide = grpc.load(PROTO_PATH).routeguide; +var protoLoader = require('@grpc/proto-loader'); +var packageDefinition = protoLoader.loadSync( + PROTO_PATH, + {keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true + }); +var routeguide = grpc.loadPackageDefinition(packageDefinition).routeguide; var COORD_FACTOR = 1e7; diff --git a/examples/node/package.json b/examples/node/package.json index 6317838295a..3af6a113fd5 100644 --- a/examples/node/package.json +++ b/examples/node/package.json @@ -2,9 +2,10 @@ "name": "grpc-examples", "version": "0.1.0", "dependencies": { + "@grpc/proto-loader": "^0.1.0", "async": "^1.5.2", "google-protobuf": "^3.0.0", - "grpc": "^1.0.0", + "grpc": "^1.11.0", "lodash": "^4.6.1", "minimist": "^1.2.0" } diff --git a/examples/python/helloworld/greeter_server_with_reflection.py b/examples/python/helloworld/greeter_server_with_reflection.py new file mode 100644 index 00000000000..5ba8782dfca --- /dev/null +++ b/examples/python/helloworld/greeter_server_with_reflection.py @@ -0,0 +1,52 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The reflection-enabled version of gRPC helloworld.Greeter server.""" + +from concurrent import futures +import time + +import grpc +from grpc_reflection.v1alpha import reflection + +import helloworld_pb2 +import helloworld_pb2_grpc + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, context): + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == '__main__': + serve() diff --git a/examples/python/multiplex/helloworld_pb2.py b/examples/python/multiplex/helloworld_pb2.py index 6665b1f6878..e18ab9acc7a 100644 --- a/examples/python/multiplex/helloworld_pb2.py +++ b/examples/python/multiplex/helloworld_pb2.py @@ -21,7 +21,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -89,6 +88,7 @@ _HELLOREPLY = _descriptor.Descriptor( DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST DESCRIPTOR.message_types_by_name['HelloReply'] = _HELLOREPLY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict( DESCRIPTOR = _HELLOREQUEST, @@ -107,123 +107,28 @@ _sym_db.RegisterMessage(HelloReply) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW')) -try: - # THESE ELEMENTS WILL BE DEPRECATED. - # Please use the generated *_pb2_grpc.py files instead. - import grpc - from grpc.framework.common import cardinality - from grpc.framework.interfaces.face import utilities as face_utilities - from grpc.beta import implementations as beta_implementations - from grpc.beta import interfaces as beta_interfaces - - - class GreeterStub(object): - """The greeting service definition. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.SayHello = channel.unary_unary( - '/helloworld.Greeter/SayHello', - request_serializer=HelloRequest.SerializeToString, - response_deserializer=HelloReply.FromString, - ) - - - class GreeterServicer(object): - """The greeting service definition. - """ - - def SayHello(self, request, context): - """Sends a greeting - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - - def add_GreeterServicer_to_server(servicer, server): - rpc_method_handlers = { - 'SayHello': grpc.unary_unary_rpc_method_handler( - servicer.SayHello, - request_deserializer=HelloRequest.FromString, - response_serializer=HelloReply.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'helloworld.Greeter', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - class BetaGreeterServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """The greeting service definition. - """ - def SayHello(self, request, context): - """Sends a greeting - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - - - class BetaGreeterStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """The greeting service definition. - """ - def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """Sends a greeting - """ - raise NotImplementedError() - SayHello.future = None - - - def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString, - } - response_serializers = { - ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString, - } - method_implementations = { - ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - - def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString, - } - response_deserializers = { - ('helloworld.Greeter', 'SayHello'): HelloReply.FromString, - } - cardinalities = { - 'SayHello': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options) -except ImportError: - pass + +_GREETER = _descriptor.ServiceDescriptor( + name='Greeter', + full_name='helloworld.Greeter', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=93, + serialized_end=166, + methods=[ + _descriptor.MethodDescriptor( + name='SayHello', + full_name='helloworld.Greeter.SayHello', + index=0, + containing_service=None, + input_type=_HELLOREQUEST, + output_type=_HELLOREPLY, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_GREETER) + +DESCRIPTOR.services_by_name['Greeter'] = _GREETER + # @@protoc_insertion_point(module_scope) diff --git a/examples/python/multiplex/helloworld_pb2_grpc.py b/examples/python/multiplex/helloworld_pb2_grpc.py index 682dc36cd89..18e07d16797 100644 --- a/examples/python/multiplex/helloworld_pb2_grpc.py +++ b/examples/python/multiplex/helloworld_pb2_grpc.py @@ -1,6 +1,5 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities import helloworld_pb2 as helloworld__pb2 diff --git a/examples/python/multiplex/route_guide_pb2.py b/examples/python/multiplex/route_guide_pb2.py index e6775eb8140..fbbc194a580 100644 --- a/examples/python/multiplex/route_guide_pb2.py +++ b/examples/python/multiplex/route_guide_pb2.py @@ -21,7 +21,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_pb=_b('\n\x11route_guide.proto\x12\nrouteguide\",\n\x05Point\x12\x10\n\x08latitude\x18\x01 \x01(\x05\x12\x11\n\tlongitude\x18\x02 \x01(\x05\"I\n\tRectangle\x12\x1d\n\x02lo\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x1d\n\x02hi\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"<\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x08location\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"A\n\tRouteNote\x12#\n\x08location\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x0f\n\x07message\x18\x02 \x01(\t\"b\n\x0cRouteSummary\x12\x13\n\x0bpoint_count\x18\x01 \x01(\x05\x12\x15\n\rfeature_count\x18\x02 \x01(\x05\x12\x10\n\x08\x64istance\x18\x03 \x01(\x05\x12\x14\n\x0c\x65lapsed_time\x18\x04 \x01(\x05\x32\x85\x02\n\nRouteGuide\x12\x36\n\nGetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n\x0cListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x00\x30\x01\x12>\n\x0bRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x01\x30\x01\x42\x36\n\x1bio.grpc.examples.routeguideB\x0fRouteGuideProtoP\x01\xa2\x02\x03RTGb\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -238,6 +237,7 @@ DESCRIPTOR.message_types_by_name['Rectangle'] = _RECTANGLE DESCRIPTOR.message_types_by_name['Feature'] = _FEATURE DESCRIPTOR.message_types_by_name['RouteNote'] = _ROUTENOTE DESCRIPTOR.message_types_by_name['RouteSummary'] = _ROUTESUMMARY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) Point = _reflection.GeneratedProtocolMessageType('Point', (_message.Message,), dict( DESCRIPTOR = _POINT, @@ -277,265 +277,55 @@ _sym_db.RegisterMessage(RouteSummary) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG')) -try: - # THESE ELEMENTS WILL BE DEPRECATED. - # Please use the generated *_pb2_grpc.py files instead. - import grpc - from grpc.framework.common import cardinality - from grpc.framework.interfaces.face import utilities as face_utilities - from grpc.beta import implementations as beta_implementations - from grpc.beta import interfaces as beta_interfaces - - - class RouteGuideStub(object): - """Interface exported by the server. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.GetFeature = channel.unary_unary( - '/routeguide.RouteGuide/GetFeature', - request_serializer=Point.SerializeToString, - response_deserializer=Feature.FromString, - ) - self.ListFeatures = channel.unary_stream( - '/routeguide.RouteGuide/ListFeatures', - request_serializer=Rectangle.SerializeToString, - response_deserializer=Feature.FromString, - ) - self.RecordRoute = channel.stream_unary( - '/routeguide.RouteGuide/RecordRoute', - request_serializer=Point.SerializeToString, - response_deserializer=RouteSummary.FromString, - ) - self.RouteChat = channel.stream_stream( - '/routeguide.RouteGuide/RouteChat', - request_serializer=RouteNote.SerializeToString, - response_deserializer=RouteNote.FromString, - ) - - - class RouteGuideServicer(object): - """Interface exported by the server. - """ - - def GetFeature(self, request, context): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ListFeatures(self, request, context): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def RecordRoute(self, request_iterator, context): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def RouteChat(self, request_iterator, context): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - - def add_RouteGuideServicer_to_server(servicer, server): - rpc_method_handlers = { - 'GetFeature': grpc.unary_unary_rpc_method_handler( - servicer.GetFeature, - request_deserializer=Point.FromString, - response_serializer=Feature.SerializeToString, - ), - 'ListFeatures': grpc.unary_stream_rpc_method_handler( - servicer.ListFeatures, - request_deserializer=Rectangle.FromString, - response_serializer=Feature.SerializeToString, - ), - 'RecordRoute': grpc.stream_unary_rpc_method_handler( - servicer.RecordRoute, - request_deserializer=Point.FromString, - response_serializer=RouteSummary.SerializeToString, - ), - 'RouteChat': grpc.stream_stream_rpc_method_handler( - servicer.RouteChat, - request_deserializer=RouteNote.FromString, - response_serializer=RouteNote.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'routeguide.RouteGuide', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - class BetaRouteGuideServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """Interface exported by the server. - """ - def GetFeature(self, request, context): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def ListFeatures(self, request, context): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def RecordRoute(self, request_iterator, context): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def RouteChat(self, request_iterator, context): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - - - class BetaRouteGuideStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """Interface exported by the server. - """ - def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - raise NotImplementedError() - GetFeature.future = None - def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - raise NotImplementedError() - def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - raise NotImplementedError() - RecordRoute.future = None - def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - raise NotImplementedError() - - - def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('routeguide.RouteGuide', 'GetFeature'): Point.FromString, - ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString, - ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString, - } - response_serializers = { - ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString, - ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString, - ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString, - } - method_implementations = { - ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature), - ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures), - ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute), - ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - - def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. +_ROUTEGUIDE = _descriptor.ServiceDescriptor( + name='RouteGuide', + full_name='routeguide.RouteGuide', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=384, + serialized_end=645, + methods=[ + _descriptor.MethodDescriptor( + name='GetFeature', + full_name='routeguide.RouteGuide.GetFeature', + index=0, + containing_service=None, + input_type=_POINT, + output_type=_FEATURE, + options=None, + ), + _descriptor.MethodDescriptor( + name='ListFeatures', + full_name='routeguide.RouteGuide.ListFeatures', + index=1, + containing_service=None, + input_type=_RECTANGLE, + output_type=_FEATURE, + options=None, + ), + _descriptor.MethodDescriptor( + name='RecordRoute', + full_name='routeguide.RouteGuide.RecordRoute', + index=2, + containing_service=None, + input_type=_POINT, + output_type=_ROUTESUMMARY, + options=None, + ), + _descriptor.MethodDescriptor( + name='RouteChat', + full_name='routeguide.RouteGuide.RouteChat', + index=3, + containing_service=None, + input_type=_ROUTENOTE, + output_type=_ROUTENOTE, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ROUTEGUIDE) + +DESCRIPTOR.services_by_name['RouteGuide'] = _ROUTEGUIDE - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString, - ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString, - ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString, - } - response_deserializers = { - ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString, - ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString, - ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString, - } - cardinalities = { - 'GetFeature': cardinality.Cardinality.UNARY_UNARY, - 'ListFeatures': cardinality.Cardinality.UNARY_STREAM, - 'RecordRoute': cardinality.Cardinality.STREAM_UNARY, - 'RouteChat': cardinality.Cardinality.STREAM_STREAM, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options) -except ImportError: - pass # @@protoc_insertion_point(module_scope) diff --git a/examples/python/multiplex/route_guide_pb2_grpc.py b/examples/python/multiplex/route_guide_pb2_grpc.py index 27b24c747db..05c1b793128 100644 --- a/examples/python/multiplex/route_guide_pb2_grpc.py +++ b/examples/python/multiplex/route_guide_pb2_grpc.py @@ -1,6 +1,5 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities import route_guide_pb2 as route__guide__pb2 diff --git a/examples/ruby/grpc-demo.gemspec b/examples/ruby/grpc-demo.gemspec index 4423fd34d42..90f75fbb2b7 100644 --- a/examples/ruby/grpc-demo.gemspec +++ b/examples/ruby/grpc-demo.gemspec @@ -18,6 +18,6 @@ Gem::Specification.new do |s| s.platform = Gem::Platform::RUBY s.add_dependency 'grpc', '~> 1.0' - + s.add_dependency 'multi_json', '~> 1.13.1' s.add_development_dependency 'bundler', '~> 1.7' end diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 1d9237bf627..ae3bd4b312f 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -236,6 +236,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/fork.h', 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/memory.h', + 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/ext/transport/chttp2/transport/bin_decoder.h', @@ -278,6 +279,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', @@ -532,6 +535,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/fork.h', 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/memory.h', + 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/lib/avl/avl.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 23edaec656d..8e60a6065f5 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -208,6 +208,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/fork.h', 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/memory.h', + 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/lib/gpr/alloc.cc', @@ -289,6 +290,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', @@ -705,6 +708,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', @@ -802,11 +807,14 @@ Pod::Spec.new do |s| 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', - 'src/cpp/ext/filters/census/grpc_context.cc', + 'src/core/ext/filters/census/grpc_context.cc', 'src/core/ext/filters/max_age/max_age_filter.cc', 'src/core/ext/filters/message_size/message_size_filter.cc', 'src/core/ext/filters/http/client_authority_filter.cc', @@ -837,6 +845,7 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/fork.h', 'src/core/lib/gprpp/manual_constructor.h', 'src/core/lib/gprpp/memory.h', + 'src/core/lib/gprpp/mutex_lock.h', 'src/core/lib/gprpp/thd.h', 'src/core/lib/profiling/timers.h', 'src/core/ext/transport/chttp2/transport/bin_decoder.h', @@ -879,6 +888,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', @@ -1109,6 +1120,7 @@ Pod::Spec.new do |s| ss.source_files = 'src/core/lib/iomgr/cfstream_handle.cc', 'src/core/lib/iomgr/endpoint_cfstream.cc', 'src/core/lib/iomgr/error_cfstream.cc', + 'src/core/lib/iomgr/iomgr_posix_cfstream.cc', 'src/core/lib/iomgr/tcp_client_cfstream.cc', 'src/core/lib/iomgr/cfstream_handle.h', 'src/core/lib/iomgr/endpoint_cfstream.h', diff --git a/grpc.def b/grpc.def index 5b98792662c..009de4e8688 100644 --- a/grpc.def +++ b/grpc.def @@ -20,6 +20,7 @@ EXPORTS grpc_completion_queue_factory_lookup grpc_completion_queue_create_for_next grpc_completion_queue_create_for_pluck + grpc_completion_queue_create_for_callback grpc_completion_queue_create grpc_completion_queue_next grpc_completion_queue_pluck @@ -42,6 +43,7 @@ EXPORTS grpc_census_call_get_context grpc_channel_get_target grpc_channel_get_info + grpc_channel_reset_connect_backoff grpc_insecure_channel_create grpc_lame_client_channel_create grpc_channel_destroy @@ -68,6 +70,7 @@ EXPORTS grpc_resource_quota_ref grpc_resource_quota_unref grpc_resource_quota_resize + grpc_resource_quota_set_max_threads grpc_resource_quota_arg_vtable grpc_channelz_get_top_channels grpc_channelz_get_channel diff --git a/grpc.gemspec b/grpc.gemspec index b69d5a7c6fc..94ce48d61eb 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -105,6 +105,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/gprpp/fork.h ) s.files += %w( src/core/lib/gprpp/manual_constructor.h ) s.files += %w( src/core/lib/gprpp/memory.h ) + s.files += %w( src/core/lib/gprpp/mutex_lock.h ) s.files += %w( src/core/lib/gprpp/thd.h ) s.files += %w( src/core/lib/profiling/timers.h ) s.files += %w( src/core/lib/gpr/alloc.cc ) @@ -222,6 +223,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.h ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.h ) s.files += %w( src/core/lib/security/security_connector/alts_security_connector.h ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots.h ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_linux.h ) s.files += %w( src/core/lib/security/security_connector/local_security_connector.h ) s.files += %w( src/core/lib/security/security_connector/security_connector.h ) s.files += %w( src/core/lib/security/transport/auth_filters.h ) @@ -642,6 +645,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.cc ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.cc ) s.files += %w( src/core/lib/security/security_connector/alts_security_connector.cc ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_fallback.cc ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_linux.cc ) s.files += %w( src/core/lib/security/security_connector/local_security_connector.cc ) s.files += %w( src/core/lib/security/security_connector/security_connector.cc ) s.files += %w( src/core/lib/security/transport/client_auth_filter.cc ) @@ -742,11 +747,14 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc ) + s.files += %w( src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc ) s.files += %w( src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc ) - s.files += %w( src/cpp/ext/filters/census/grpc_context.cc ) + s.files += %w( src/core/ext/filters/census/grpc_context.cc ) s.files += %w( src/core/ext/filters/max_age/max_age_filter.cc ) s.files += %w( src/core/ext/filters/message_size/message_size_filter.cc ) s.files += %w( src/core/ext/filters/http/client_authority_filter.cc ) diff --git a/grpc.gyp b/grpc.gyp index e1485efa058..a36998bcb3b 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -472,6 +472,8 @@ 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', @@ -572,11 +574,14 @@ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', - 'src/cpp/ext/filters/census/grpc_context.cc', + 'src/core/ext/filters/census/grpc_context.cc', 'src/core/ext/filters/max_age/max_age_filter.cc', 'src/core/ext/filters/message_size/message_size_filter.cc', 'src/core/ext/filters/http/client_authority_filter.cc', @@ -1287,8 +1292,11 @@ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', 'src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc', @@ -1305,7 +1313,7 @@ 'third_party/nanopb/pb_encode.c', 'src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc', 'src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc', - 'src/cpp/ext/filters/census/grpc_context.cc', + 'src/core/ext/filters/census/grpc_context.cc', 'src/core/ext/filters/max_age/max_age_filter.cc', 'src/core/ext/filters/message_size/message_size_filter.cc', 'src/core/ext/filters/http/client_authority_filter.cc', diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index f0eb2c0121b..897b89851a0 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -60,6 +60,8 @@ GRPCAPI void grpc_register_plugin(void (*init)(void), void (*destroy)(void)); /** Initialize the grpc library. + After it's called, a matching invocation to grpc_shutdown() is expected. + It is not safe to call any other grpc functions before calling this. (To avoid overhead, little checking is done, and some things may work. We do not warrant that they will continue to do so in future revisions of this @@ -68,6 +70,9 @@ GRPCAPI void grpc_init(void); /** Shut down the grpc library. + Before it's called, there should haven been a matching invocation to + grpc_init(). + No memory is used by grpc after this call returns, nor are any instructions executing within the grpc library. Prior to calling, all application owned grpc objects must have been @@ -96,6 +101,12 @@ GRPCAPI grpc_completion_queue* grpc_completion_queue_create_for_next( GRPCAPI grpc_completion_queue* grpc_completion_queue_create_for_pluck( void* reserved); +/** Helper function to create a completion queue with grpc_cq_completion_type + of GRPC_CQ_CALLBACK and grpc_cq_polling_type of GRPC_CQ_DEFAULT_POLLING. + This function is experimental. */ +GRPCAPI grpc_completion_queue* grpc_completion_queue_create_for_callback( + void* shutdown_callback, void* reserved); + /** Create a completion queue */ GRPCAPI grpc_completion_queue* grpc_completion_queue_create( const grpc_completion_queue_factory* factory, @@ -269,6 +280,11 @@ GRPCAPI char* grpc_channel_get_target(grpc_channel* channel); GRPCAPI void grpc_channel_get_info(grpc_channel* channel, const grpc_channel_info* channel_info); +/** EXPERIMENTAL. Resets the channel's connect backoff. + TODO(roth): When we see whether this proves useful, either promote + to non-experimental or remove it. */ +GRPCAPI void grpc_channel_reset_connect_backoff(grpc_channel* channel); + /** Create a client channel to 'target'. Additional channel level configuration MAY be provided by grpc_channel_args, though the expectation is that most clients will want to simply pass NULL. The user data in 'args' need only @@ -450,6 +466,10 @@ GRPCAPI void grpc_resource_quota_unref(grpc_resource_quota* resource_quota); GRPCAPI void grpc_resource_quota_resize(grpc_resource_quota* resource_quota, size_t new_size); +/** Update the size of the maximum number of threads allowed */ +GRPCAPI void grpc_resource_quota_set_max_threads( + grpc_resource_quota* resource_quota, int new_max_threads); + /** Fetch a vtable for a grpc_channel_arg that points to a grpc_resource_quota */ GRPCAPI const grpc_arg_pointer_vtable* grpc_resource_quota_arg_vtable(void); diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 102d20fcf98..02d87a493a1 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -587,12 +587,6 @@ GRPCAPI grpc_server_credentials* grpc_alts_server_credentials_create( /** --- Local channel/server credentials --- **/ -/** - * Type of local connection for which local channel/server credentials will be - * applied. It only supports UDS for now. - */ -typedef enum { UDS = 0 } grpc_local_connect_type; - /** * This method creates a local channel credential object. It is used for * experimental purpose for now and subject to change. diff --git a/include/grpc/grpc_security_constants.h b/include/grpc/grpc_security_constants.h index 92580ea35e9..944a1e927f0 100644 --- a/include/grpc/grpc_security_constants.h +++ b/include/grpc/grpc_security_constants.h @@ -100,6 +100,12 @@ typedef enum { GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY } grpc_ssl_client_certificate_request_type; +/** + * Type of local connection for which local channel/server credentials will be + * applied. It only supports UDS for now. + */ +typedef enum { UDS = 0 } grpc_local_connect_type; + #ifdef __cplusplus } #endif diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 5fd080c48b2..b5353c1dea8 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -651,10 +651,16 @@ typedef enum { GRPC_CQ_NEXT, /** Events are popped out by calling grpc_completion_queue_pluck() API ONLY*/ - GRPC_CQ_PLUCK + GRPC_CQ_PLUCK, + + /** EXPERIMENTAL: Events trigger a callback specified as the tag */ + GRPC_CQ_CALLBACK } grpc_cq_completion_type; -#define GRPC_CQ_CURRENT_VERSION 1 +/* The upgrade to version 2 is currently experimental. */ + +#define GRPC_CQ_CURRENT_VERSION 2 +#define GRPC_CQ_VERSION_MINIMUM_FOR_CALLBACKABLE 2 typedef struct grpc_completion_queue_attributes { /** The version number of this structure. More fields might be added to this structure in future. */ @@ -663,6 +669,15 @@ typedef struct grpc_completion_queue_attributes { grpc_cq_completion_type cq_completion_type; grpc_cq_polling_type cq_polling_type; + + /* END OF VERSION 1 CQ ATTRIBUTES */ + + /* EXPERIMENTAL: START OF VERSION 2 CQ ATTRIBUTES */ + /** When creating a callbackable CQ, pass in a functor to get invoked when + * shutdown is complete */ + void* cq_shutdown_cb; + + /* END OF VERSION 2 CQ ATTRIBUTES */ } grpc_completion_queue_attributes; /** The completion queue factory structure is opaque to the callers of grpc */ diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 01ce5f03e96..8d9bd832855 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -282,6 +282,47 @@ #else /* _LP64 */ #define GPR_ARCH_32 1 #endif /* _LP64 */ +#elif defined(__sun) && defined(__SVR4) +#define GPR_PLATFORM_STRING "solaris" +#define GPR_SOLARIS 1 +#define GPR_CPU_POSIX 1 +#define GPR_GCC_ATOMIC 1 +#define GPR_GCC_TLS 1 +#define GPR_POSIX_LOG 1 +#define GPR_POSIX_ENV 1 +#define GPR_POSIX_TMPFILE 1 +#define GPR_POSIX_STRING 1 +#define GPR_POSIX_SUBPROCESS 1 +#define GPR_POSIX_SYNC 1 +#define GPR_POSIX_TIME 1 +#define GPR_GETPID_IN_UNISTD_H 1 +#ifdef _LP64 +#define GPR_ARCH_64 1 +#else /* _LP64 */ +#define GPR_ARCH_32 1 +#endif /* _LP64 */ +#elif defined(_AIX) +#define GPR_PLATFORM_STRING "aix" +#ifndef _ALL_SOURCE +#define _ALL_SOURCE +#endif +#define GPR_AIX 1 +#define GPR_CPU_POSIX 1 +#define GPR_GCC_ATOMIC 1 +#define GPR_GCC_TLS 1 +#define GPR_POSIX_LOG 1 +#define GPR_POSIX_ENV 1 +#define GPR_POSIX_TMPFILE 1 +#define GPR_POSIX_STRING 1 +#define GPR_POSIX_SUBPROCESS 1 +#define GPR_POSIX_SYNC 1 +#define GPR_POSIX_TIME 1 +#define GPR_GETPID_IN_UNISTD_H 1 +#ifdef _LP64 +#define GPR_ARCH_64 1 +#else /* _LP64 */ +#define GPR_ARCH_32 1 +#endif /* _LP64 */ #elif defined(__native_client__) #define GPR_PLATFORM_STRING "nacl" #ifndef _BSD_SOURCE @@ -420,12 +461,8 @@ typedef unsigned __int64 uint64_t; #define GPR_MAX_ALIGNMENT 16 #ifndef GRPC_ARES -#ifdef GPR_WINDOWS -#define GRPC_ARES 0 -#else #define GRPC_ARES 1 #endif -#endif #ifndef GRPC_MUST_USE_RESULT #if defined(__GNUC__) && !defined(__MINGW32__) diff --git a/include/grpc/support/sync.h b/include/grpc/support/sync.h index 91d1fa79b5d..da820dece5e 100644 --- a/include/grpc/support/sync.h +++ b/include/grpc/support/sync.h @@ -277,22 +277,6 @@ GPRAPI intptr_t gpr_stats_read(const gpr_stats_counter* c); #ifdef __cplusplus } // extern "C" - -namespace grpc_core { - -class mu_guard { - public: - mu_guard(gpr_mu* mu) : mu_(mu) { gpr_mu_lock(mu); } - ~mu_guard() { gpr_mu_unlock(mu_); } - - mu_guard(const mu_guard&) = delete; - mu_guard& operator=(const mu_guard&) = delete; - - private: - gpr_mu* const mu_; -}; - -} // namespace grpc_core #endif #endif /* GRPC_SUPPORT_SYNC_H */ diff --git a/include/grpcpp/channel.h b/include/grpcpp/channel.h index 4b45d5382cd..fed02bf7bc7 100644 --- a/include/grpcpp/channel.h +++ b/include/grpcpp/channel.h @@ -30,6 +30,14 @@ struct grpc_channel; namespace grpc { + +namespace experimental { +/// Resets the channel's connection backoff. +/// TODO(roth): Once we see whether this proves useful, either create a gRFC +/// and change this to be a method of the Channel class, or remove it. +void ChannelResetConnectionBackoff(Channel* channel); +} // namespace experimental + /// Channels represent a connection to an endpoint. Created by \a CreateChannel. class Channel final : public ChannelInterface, public internal::CallHook, @@ -52,6 +60,7 @@ class Channel final : public ChannelInterface, private: template friend class internal::BlockingUnaryCallImpl; + friend void experimental::ChannelResetConnectionBackoff(Channel* channel); friend std::shared_ptr CreateChannelInternal( const grpc::string& host, grpc_channel* c_channel); Channel(const grpc::string& host, grpc_channel* c_channel); diff --git a/include/grpcpp/impl/codegen/async_generic_service.h b/include/grpcpp/impl/codegen/async_generic_service.h index 957bb776f1d..2a0e1b40881 100644 --- a/include/grpcpp/impl/codegen/async_generic_service.h +++ b/include/grpcpp/impl/codegen/async_generic_service.h @@ -52,12 +52,12 @@ class GenericServerContext final : public ServerContext { // ServerBuilder builder; // auto cq = builder.AddCompletionQueue(); // AsyncGenericService generic_service; -// builder.RegisterAsyncGeneicService(&generic_service); +// builder.RegisterAsyncGenericService(&generic_service); // auto server = builder.BuildAndStart(); // // // request a new call // GenericServerContext context; -// GenericAsyncReaderWriter stream; +// GenericServerAsyncReaderWriter stream; // generic_service.RequestCall(&context, &stream, cq.get(), cq.get(), tag); // // When tag is retrieved from cq->Next(), context.method() can be used to look diff --git a/include/grpcpp/impl/codegen/byte_buffer.h b/include/grpcpp/impl/codegen/byte_buffer.h index 86c047ebe7a..8cc51581152 100644 --- a/include/grpcpp/impl/codegen/byte_buffer.h +++ b/include/grpcpp/impl/codegen/byte_buffer.h @@ -45,6 +45,8 @@ template class RpcMethodHandler; template class ServerStreamingHandler; +template +class ErrorMethodHandler; template class DeserializeFuncType; class GrpcByteBufferPeer; @@ -144,6 +146,8 @@ class ByteBuffer final { friend class internal::RpcMethodHandler; template friend class internal::ServerStreamingHandler; + template + friend class internal::ErrorMethodHandler; template friend class internal::DeserializeFuncType; friend class ProtoBufferReader; diff --git a/include/grpcpp/impl/codegen/client_unary_call.h b/include/grpcpp/impl/codegen/client_unary_call.h index a37a81b75b8..e4e8364e073 100644 --- a/include/grpcpp/impl/codegen/client_unary_call.h +++ b/include/grpcpp/impl/codegen/client_unary_call.h @@ -50,8 +50,8 @@ class BlockingUnaryCallImpl { ClientContext* context, const InputMessage& request, OutputMessage* result) { CompletionQueue cq(grpc_completion_queue_attributes{ - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, - GRPC_CQ_DEFAULT_POLLING}); // Pluckable completion queue + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, GRPC_CQ_DEFAULT_POLLING, + nullptr}); // Pluckable completion queue Call call(channel->CreateCall(method, context, &cq)); CallOpSet, diff --git a/include/grpcpp/impl/codegen/completion_queue.h b/include/grpcpp/impl/codegen/completion_queue.h index 5819e068bad..3f7d4fb765f 100644 --- a/include/grpcpp/impl/codegen/completion_queue.h +++ b/include/grpcpp/impl/codegen/completion_queue.h @@ -78,9 +78,10 @@ template class ServerStreamingHandler; template class BidiStreamingHandler; -class UnknownMethodHandler; template class TemplatedBidiStreamingHandler; +template +class ErrorMethodHandler; template class BlockingUnaryCallImpl; } // namespace internal @@ -97,7 +98,8 @@ class CompletionQueue : private GrpcLibraryCodegen { /// instance. CompletionQueue() : CompletionQueue(grpc_completion_queue_attributes{ - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING}) {} + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING, + nullptr}) {} /// Wrap \a take, taking ownership of the instance. /// @@ -264,7 +266,8 @@ class CompletionQueue : private GrpcLibraryCodegen { friend class ::grpc::internal::ServerStreamingHandler; template friend class ::grpc::internal::TemplatedBidiStreamingHandler; - friend class ::grpc::internal::UnknownMethodHandler; + template + friend class ::grpc::internal::ErrorMethodHandler; friend class ::grpc::Server; friend class ::grpc::ServerContext; friend class ::grpc::ServerInterface; @@ -376,7 +379,7 @@ class ServerCompletionQueue : public CompletionQueue { /// frequently polled. ServerCompletionQueue(grpc_cq_polling_type polling_type) : CompletionQueue(grpc_completion_queue_attributes{ - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, polling_type}), + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, polling_type, nullptr}), polling_type_(polling_type) {} grpc_cq_polling_type polling_type_; diff --git a/include/grpcpp/impl/codegen/method_handler_impl.h b/include/grpcpp/impl/codegen/method_handler_impl.h index 851aa2a024b..53117f941b4 100644 --- a/include/grpcpp/impl/codegen/method_handler_impl.h +++ b/include/grpcpp/impl/codegen/method_handler_impl.h @@ -272,12 +272,14 @@ class SplitServerStreamingHandler ServerSplitStreamer, false>(func) {} }; -/// Handle unknown method by returning UNIMPLEMENTED error. -class UnknownMethodHandler : public MethodHandler { +/// General method handler class for errors that prevent real method use +/// e.g., handle unknown method by returning UNIMPLEMENTED error. +template +class ErrorMethodHandler : public MethodHandler { public: template static void FillOps(ServerContext* context, T* ops) { - Status status(StatusCode::UNIMPLEMENTED, ""); + Status status(code, ""); if (!context->sent_initial_metadata_) { ops->SendInitialMetadata(context->initial_metadata_, context->initial_metadata_flags()); @@ -294,9 +296,18 @@ class UnknownMethodHandler : public MethodHandler { FillOps(param.server_context, &ops); param.call->PerformOps(&ops); param.call->cq()->Pluck(&ops); + // We also have to destroy any request payload in the handler parameter + ByteBuffer* payload = param.request.bbuf_ptr(); + if (payload != nullptr) { + payload->Clear(); + } } }; +typedef ErrorMethodHandler UnknownMethodHandler; +typedef ErrorMethodHandler + ResourceExhaustedHandler; + } // namespace internal } // namespace grpc diff --git a/include/grpcpp/impl/codegen/server_context.h b/include/grpcpp/impl/codegen/server_context.h index 153b404d9e1..6314364db67 100644 --- a/include/grpcpp/impl/codegen/server_context.h +++ b/include/grpcpp/impl/codegen/server_context.h @@ -63,9 +63,10 @@ template class ServerStreamingHandler; template class BidiStreamingHandler; -class UnknownMethodHandler; template class TemplatedBidiStreamingHandler; +template +class ErrorMethodHandler; class Call; } // namespace internal @@ -226,6 +227,8 @@ class ServerContext { /// Async only. Has to be called before the rpc starts. /// Returns the tag in completion queue when the rpc finishes. /// IsCancelled() can then be called to check whether the rpc was cancelled. + /// TODO(vjpai): Fix this so that the tag is returned even if the call never + /// starts (https://github.com/grpc/grpc/issues/10136). void AsyncNotifyWhenDone(void* tag) { has_notify_when_done_tag_ = true; async_notify_when_done_tag_ = tag; @@ -262,7 +265,8 @@ class ServerContext { friend class ::grpc::internal::ServerStreamingHandler; template friend class ::grpc::internal::TemplatedBidiStreamingHandler; - friend class ::grpc::internal::UnknownMethodHandler; + template + friend class internal::ErrorMethodHandler; friend class ::grpc::ClientContext; /// Prevent copying. diff --git a/include/grpcpp/impl/codegen/sync_stream.h b/include/grpcpp/impl/codegen/sync_stream.h index 7152eaf41fe..cbfcf25d0a4 100644 --- a/include/grpcpp/impl/codegen/sync_stream.h +++ b/include/grpcpp/impl/codegen/sync_stream.h @@ -243,8 +243,8 @@ class ClientReader final : public ClientReaderInterface { ClientContext* context, const W& request) : context_(context), cq_(grpc_completion_queue_attributes{ - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, - GRPC_CQ_DEFAULT_POLLING}), // Pluckable cq + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, GRPC_CQ_DEFAULT_POLLING, + nullptr}), // Pluckable cq call_(channel->CreateCall(method, context, &cq_)) { ::grpc::internal::CallOpSet<::grpc::internal::CallOpSendInitialMetadata, ::grpc::internal::CallOpSendMessage, @@ -377,8 +377,8 @@ class ClientWriter : public ClientWriterInterface { ClientContext* context, R* response) : context_(context), cq_(grpc_completion_queue_attributes{ - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, - GRPC_CQ_DEFAULT_POLLING}), // Pluckable cq + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, GRPC_CQ_DEFAULT_POLLING, + nullptr}), // Pluckable cq call_(channel->CreateCall(method, context, &cq_)) { finish_ops_.RecvMessage(response); finish_ops_.AllowNoMessage(); @@ -551,8 +551,8 @@ class ClientReaderWriter final : public ClientReaderWriterInterface { ClientContext* context) : context_(context), cq_(grpc_completion_queue_attributes{ - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, - GRPC_CQ_DEFAULT_POLLING}), // Pluckable cq + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_PLUCK, GRPC_CQ_DEFAULT_POLLING, + nullptr}), // Pluckable cq call_(channel->CreateCall(method, context, &cq_)) { if (!context_->initial_metadata_corked_) { ::grpc::internal::CallOpSet<::grpc::internal::CallOpSendInitialMetadata> diff --git a/include/grpcpp/opencensus.h b/include/grpcpp/opencensus.h index 7e5d1dfeb41..07a13339863 100644 --- a/include/grpcpp/opencensus.h +++ b/include/grpcpp/opencensus.h @@ -19,6 +19,12 @@ #ifndef GRPCPP_OPENCENSUS_H #define GRPCPP_OPENCENSUS_H +#ifndef GRPC_BAZEL_BUILD +#error OpenCensus for gRPC is only supported when building with bazel. +#endif + +#include "opencensus/trace/span.h" + namespace grpc { // These symbols in this file will not be included in the binary unless // grpc_opencensus_plugin build target was added as a dependency. At the moment @@ -36,6 +42,11 @@ void RegisterOpenCensusPlugin(); // ViewDescriptors below. void RegisterOpenCensusViewsForExport(); +class ServerContext; + +// Returns the tracing Span for the current RPC. +::opencensus::trace::Span GetSpanFromServerContext(ServerContext* context); + } // namespace grpc #endif // GRPCPP_OPENCENSUS_H diff --git a/include/grpcpp/resource_quota.h b/include/grpcpp/resource_quota.h index 554437a40d1..50bd1cb849a 100644 --- a/include/grpcpp/resource_quota.h +++ b/include/grpcpp/resource_quota.h @@ -26,10 +26,10 @@ struct grpc_resource_quota; namespace grpc { -/// ResourceQuota represents a bound on memory usage by the gRPC library. -/// A ResourceQuota can be attached to a server (via \a ServerBuilder), +/// ResourceQuota represents a bound on memory and thread usage by the gRPC +/// library. A ResourceQuota can be attached to a server (via \a ServerBuilder), /// or a client channel (via \a ChannelArguments). -/// gRPC will attempt to keep memory used by all attached entities +/// gRPC will attempt to keep memory and threads used by all attached entities /// below the ResourceQuota bound. class ResourceQuota final : private GrpcLibraryCodegen { public: @@ -44,6 +44,16 @@ class ResourceQuota final : private GrpcLibraryCodegen { /// No time bound is given for this to occur however. ResourceQuota& Resize(size_t new_size); + /// Set the max number of threads that can be allocated from this + /// ResourceQuota object. + /// + /// If the new_max_threads value is smaller than the current value, no new + /// threads are allocated until the number of active threads fall below + /// new_max_threads. There is no time bound on when this may happen i.e none + /// of the current threads are forcefully destroyed and all threads run their + /// normal course. + ResourceQuota& SetMaxThreads(int new_max_threads); + grpc_resource_quota* c_resource_quota() const { return impl_; } private: diff --git a/include/grpcpp/security/credentials.h b/include/grpcpp/security/credentials.h index 36d95d1b426..bfadc15df57 100644 --- a/include/grpcpp/security/credentials.h +++ b/include/grpcpp/security/credentials.h @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -234,6 +235,10 @@ struct AltsCredentialsOptions { std::shared_ptr AltsCredentials( const AltsCredentialsOptions& options); +/// Builds Local Credentials. +std::shared_ptr LocalCredentials( + grpc_local_connect_type type); + } // namespace experimental } // namespace grpc diff --git a/include/grpcpp/security/server_credentials.h b/include/grpcpp/security/server_credentials.h index cf57e275f59..bd00a0a1737 100644 --- a/include/grpcpp/security/server_credentials.h +++ b/include/grpcpp/security/server_credentials.h @@ -97,6 +97,10 @@ struct AltsServerCredentialsOptions { std::shared_ptr AltsServerCredentials( const AltsServerCredentialsOptions& options); +/// Builds Local ServerCredentials. +std::shared_ptr LocalServerCredentials( + grpc_local_connect_type type); + } // namespace experimental } // namespace grpc diff --git a/include/grpcpp/server.h b/include/grpcpp/server.h index 81c3907f86d..72544c0f0bc 100644 --- a/include/grpcpp/server.h +++ b/include/grpcpp/server.h @@ -120,6 +120,10 @@ class Server : public ServerInterface, private GrpcLibraryCodegen { int AddListeningPort(const grpc::string& addr, ServerCredentials* creds) override; + /// NOTE: This is *NOT* a public API. The server constructors are supposed to + /// be used by \a ServerBuilder class only. The constructor will be made + /// 'private' very soon. + /// /// Server constructors. To be used by \a ServerBuilder only. /// /// \param max_message_size Maximum message length that the channel can @@ -144,7 +148,8 @@ class Server : public ServerInterface, private GrpcLibraryCodegen { Server(int max_message_size, ChannelArguments* args, std::shared_ptr>> sync_server_cqs, - int min_pollers, int max_pollers, int sync_cq_timeout_msec); + int min_pollers, int max_pollers, int sync_cq_timeout_msec, + grpc_resource_quota* server_rq = nullptr); /// Start the server. /// @@ -218,6 +223,9 @@ class Server : public ServerInterface, private GrpcLibraryCodegen { std::unique_ptr health_check_service_; bool health_check_service_disabled_; + + // A special handler for resource exhausted in sync case + std::unique_ptr resource_exhausted_handler_; }; } // namespace grpc diff --git a/package.xml b/package.xml index 7f71536b1d0..9d202761f20 100644 --- a/package.xml +++ b/package.xml @@ -110,6 +110,7 @@ + @@ -227,6 +228,8 @@ + + @@ -647,6 +650,8 @@ + + @@ -747,11 +752,14 @@ + + + - + diff --git a/requirements.bazel.txt b/requirements.bazel.txt new file mode 100644 index 00000000000..16f31f9e94b --- /dev/null +++ b/requirements.bazel.txt @@ -0,0 +1,10 @@ +# GRPC Python setup requirements +coverage>=4.0 +cython==0.28.3 +enum34>=1.0.4 +protobuf>=3.5.0.post1 +six>=1.10 +wheel>=0.29 +futures>=2.2.0 +google-auth>=1.0.0 +oauth2client==4.1.0 diff --git a/src/cpp/ext/filters/census/grpc_context.cc b/src/core/ext/filters/census/grpc_context.cc similarity index 100% rename from src/cpp/ext/filters/census/grpc_context.cc rename to src/core/ext/filters/census/grpc_context.cc diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 024c9d737e8..b06f09d8c7b 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -571,15 +571,32 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { if (op->send_ping.on_initiate != nullptr || op->send_ping.on_ack != nullptr) { if (chand->lb_policy == nullptr) { - GRPC_CLOSURE_SCHED( - op->send_ping.on_initiate, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing")); - GRPC_CLOSURE_SCHED( - op->send_ping.on_ack, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing")); + grpc_error* error = + GRPC_ERROR_CREATE_FROM_STATIC_STRING("Ping with no load balancing"); + GRPC_CLOSURE_SCHED(op->send_ping.on_initiate, GRPC_ERROR_REF(error)); + GRPC_CLOSURE_SCHED(op->send_ping.on_ack, error); } else { - chand->lb_policy->PingOneLocked(op->send_ping.on_initiate, - op->send_ping.on_ack); + grpc_error* error = GRPC_ERROR_NONE; + grpc_core::LoadBalancingPolicy::PickState pick_state; + pick_state.initial_metadata = nullptr; + pick_state.initial_metadata_flags = 0; + pick_state.on_complete = nullptr; + memset(&pick_state.subchannel_call_context, 0, + sizeof(pick_state.subchannel_call_context)); + pick_state.user_data = nullptr; + // Pick must return synchronously, because pick_state.on_complete is null. + GPR_ASSERT(chand->lb_policy->PickLocked(&pick_state, &error)); + if (pick_state.connected_subchannel != nullptr) { + pick_state.connected_subchannel->Ping(op->send_ping.on_initiate, + op->send_ping.on_ack); + } else { + if (error == GRPC_ERROR_NONE) { + error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "LB policy dropped call on ping"); + } + GRPC_CLOSURE_SCHED(op->send_ping.on_initiate, GRPC_ERROR_REF(error)); + GRPC_CLOSURE_SCHED(op->send_ping.on_ack, error); + } op->bind_pollset = nullptr; } op->send_ping.on_initiate = nullptr; @@ -605,6 +622,17 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { } GRPC_ERROR_UNREF(op->disconnect_with_error); } + + if (op->reset_connect_backoff) { + if (chand->resolver != nullptr) { + chand->resolver->ResetBackoffLocked(); + chand->resolver->RequestReresolutionLocked(); + } + if (chand->lb_policy != nullptr) { + chand->lb_policy->ResetBackoffLocked(); + } + } + GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "start_transport_op"); GRPC_CLOSURE_SCHED(op->on_consumed, GRPC_ERROR_NONE); @@ -2684,14 +2712,15 @@ class LbPicker { grpc_combiner_scheduler(chand->combiner)); calld->pick.on_complete = &calld->pick_closure; GRPC_CALL_STACK_REF(calld->owning_call, "pick_callback"); - const bool pick_done = chand->lb_policy->PickLocked(&calld->pick); + grpc_error* error = GRPC_ERROR_NONE; + const bool pick_done = chand->lb_policy->PickLocked(&calld->pick, &error); if (GPR_LIKELY(pick_done)) { // Pick completed synchronously. if (grpc_client_channel_trace.enabled()) { gpr_log(GPR_INFO, "chand=%p calld=%p: pick completed synchronously", chand, calld); } - pick_done_locked(elem, GRPC_ERROR_NONE); + pick_done_locked(elem, error); GRPC_CALL_STACK_UNREF(calld->owning_call, "pick_callback"); } else { // Pick will be returned asynchronously. diff --git a/src/core/ext/filters/client_channel/client_channel_channelz.cc b/src/core/ext/filters/client_channel/client_channel_channelz.cc index 4c9c9a6bd66..86c765df521 100644 --- a/src/core/ext/filters/client_channel/client_channel_channelz.cc +++ b/src/core/ext/filters/client_channel/client_channel_channelz.cc @@ -105,8 +105,8 @@ grpc_arg ClientChannelNode::CreateChannelArg() { RefCountedPtr ClientChannelNode::MakeClientChannelNode( grpc_channel* channel, size_t channel_tracer_max_nodes, bool is_top_level_channel) { - return MakePolymorphicRefCounted( - channel, channel_tracer_max_nodes, is_top_level_channel); + return MakeRefCounted(channel, channel_tracer_max_nodes, + is_top_level_channel); } } // namespace channelz diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 3150df8847b..3c0a9c11180 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -71,6 +71,7 @@ class LoadBalancingPolicy /// Storage for LB token in \a initial_metadata, or nullptr if not used. grpc_linked_mdelem lb_token_mdelem_storage; /// Closure to run when pick is complete, if not completed synchronously. + /// If null, pick will fail if a result is not available synchronously. grpc_closure* on_complete; /// Will be set to the selected subchannel, or nullptr on failure or when /// the LB policy decides to drop the call. @@ -99,10 +100,15 @@ class LoadBalancingPolicy /// Finds an appropriate subchannel for a call, based on data in \a pick. /// \a pick must remain alive until the pick is complete. /// - /// If the pick succeeds and a result is known immediately, returns true. - /// Otherwise, \a pick->on_complete will be invoked once the pick is - /// complete with its error argument set to indicate success or failure. - virtual bool PickLocked(PickState* pick) GRPC_ABSTRACT; + /// If a result is known immediately, returns true, setting \a *error + /// upon failure. Otherwise, \a pick->on_complete will be invoked once + /// the pick is complete with its error argument set to indicate success + /// or failure. + /// + /// If \a pick->on_complete is null and no result is known immediately, + /// a synchronous failure will be returned (i.e., \a *error will be + /// set and true will be returned). + virtual bool PickLocked(PickState* pick, grpc_error** error) GRPC_ABSTRACT; /// Cancels \a pick. /// The \a on_complete callback of the pending pick will be invoked with @@ -133,18 +139,15 @@ class LoadBalancingPolicy virtual void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) GRPC_ABSTRACT; - /// Performs a connected subchannel ping via \a ConnectedSubchannel::Ping() - /// against one of the connected subchannels managed by the policy. - /// Note: This is intended only for use in tests. - virtual void PingOneLocked(grpc_closure* on_initiate, - grpc_closure* on_ack) GRPC_ABSTRACT; - /// Tries to enter a READY connectivity state. /// TODO(roth): As part of restructuring how we handle IDLE state, /// consider whether this method is still needed. virtual void ExitIdleLocked() GRPC_ABSTRACT; - /// populates child_subchannels and child_channels with the uuids of this + /// Resets connection backoff. + virtual void ResetBackoffLocked() GRPC_ABSTRACT; + + /// Populates child_subchannels and child_channels with the uuids of this /// LB policy's referenced children. This is not invoked from the /// client_channel's combiner. The implementation is responsible for /// providing its own synchronization. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 959c7441a37..25b0149393e 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -92,6 +92,7 @@ #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/manual_constructor.h" #include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/combiner.h" @@ -123,7 +124,7 @@ class GrpcLb : public LoadBalancingPolicy { GrpcLb(const grpc_lb_addresses* addresses, const Args& args); void UpdateLocked(const grpc_channel_args& args) override; - bool PickLocked(PickState* pick) override; + bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq, @@ -133,8 +134,8 @@ class GrpcLb : public LoadBalancingPolicy { grpc_connectivity_state CheckConnectivityLocked( grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; - void PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) override; @@ -167,13 +168,6 @@ class GrpcLb : public LoadBalancingPolicy { PendingPick* next = nullptr; }; - /// A linked list of pending pings waiting for the RR policy to be created. - struct PendingPing { - grpc_closure* on_initiate; - grpc_closure* on_ack; - PendingPing* next = nullptr; - }; - /// Contains a call to the LB server and all the data related to the call. class BalancerCallState : public InternallyRefCountedWithTracing { @@ -272,14 +266,12 @@ class GrpcLb : public LoadBalancingPolicy { void AddPendingPick(PendingPick* pp); static void OnPendingPickComplete(void* arg, grpc_error* error); - // Pending ping methods. - void AddPendingPing(grpc_closure* on_initiate, grpc_closure* on_ack); - // Methods for dealing with the RR policy. void CreateOrUpdateRoundRobinPolicyLocked(); grpc_channel_args* CreateRoundRobinPolicyArgsLocked(); void CreateRoundRobinPolicyLocked(const Args& args); - bool PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp); + bool PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp, + grpc_error** error); void UpdateConnectivityStateFromRoundRobinPolicyLocked( grpc_error* rr_state_error); static void OnRoundRobinConnectivityChangedLocked(void* arg, @@ -342,9 +334,8 @@ class GrpcLb : public LoadBalancingPolicy { grpc_timer lb_fallback_timer_; grpc_closure lb_on_fallback_; - // Pending picks and pings that are waiting on the RR policy's connectivity. + // Pending picks that are waiting on the RR policy's connectivity. PendingPick* pending_picks_ = nullptr; - PendingPing* pending_pings_ = nullptr; // The RR policy to use for the backends. OrphanablePtr rr_policy_; @@ -1080,7 +1071,6 @@ GrpcLb::GrpcLb(const grpc_lb_addresses* addresses, GrpcLb::~GrpcLb() { GPR_ASSERT(pending_picks_ == nullptr); - GPR_ASSERT(pending_pings_ == nullptr); gpr_mu_destroy(&lb_channel_mu_); gpr_free((void*)server_name_); grpc_channel_args_destroy(args_); @@ -1126,14 +1116,6 @@ void GrpcLb::ShutdownLocked() { // Note: pp is deleted in this callback. GRPC_CLOSURE_SCHED(&pp->on_complete, GRPC_ERROR_REF(error)); } - // Clear pending pings. - PendingPing* pping; - while ((pping = pending_pings_) != nullptr) { - pending_pings_ = pping->next; - GRPC_CLOSURE_SCHED(pping->on_initiate, GRPC_ERROR_REF(error)); - GRPC_CLOSURE_SCHED(pping->on_ack, GRPC_ERROR_REF(error)); - Delete(pping); - } GRPC_ERROR_UNREF(error); } @@ -1147,9 +1129,10 @@ void GrpcLb::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { pending_picks_ = pp->next; pp->pick->on_complete = pp->original_on_complete; pp->pick->user_data = nullptr; - if (new_policy->PickLocked(pp->pick)) { + grpc_error* error = GRPC_ERROR_NONE; + if (new_policy->PickLocked(pp->pick, &error)) { // Synchronous return; schedule closure. - GRPC_CLOSURE_SCHED(pp->pick->on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pp->pick->on_complete, error); } Delete(pp); } @@ -1233,63 +1216,51 @@ void GrpcLb::ExitIdleLocked() { } } -bool GrpcLb::PickLocked(PickState* pick) { +void GrpcLb::ResetBackoffLocked() { + if (lb_channel_ != nullptr) { + grpc_channel_reset_connect_backoff(lb_channel_); + } + if (rr_policy_ != nullptr) { + rr_policy_->ResetBackoffLocked(); + } +} + +bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) { PendingPick* pp = PendingPickCreate(pick); bool pick_done = false; if (rr_policy_ != nullptr) { - const grpc_connectivity_state rr_connectivity_state = - rr_policy_->CheckConnectivityLocked(nullptr); - // The RR policy may have transitioned to SHUTDOWN but the callback - // registered to capture this event (on_rr_connectivity_changed_) may not - // have been invoked yet. We need to make sure we aren't trying to pick - // from an RR policy instance that's in shutdown. - if (rr_connectivity_state == GRPC_CHANNEL_SHUTDOWN) { + if (grpc_lb_glb_trace.enabled()) { + gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", this, + rr_policy_.get()); + } + pick_done = + PickFromRoundRobinPolicyLocked(false /* force_async */, pp, error); + } else { // rr_policy_ == NULL + if (pick->on_complete == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No pick result available but synchronous result required."); + pick_done = true; + } else { if (grpc_lb_glb_trace.enabled()) { gpr_log(GPR_INFO, - "[grpclb %p] NOT picking from from RR %p: RR conn state=%s", - this, rr_policy_.get(), - grpc_connectivity_state_name(rr_connectivity_state)); + "[grpclb %p] No RR policy. Adding to grpclb's pending picks", + this); } AddPendingPick(pp); - pick_done = false; - } else { // RR not in shutdown - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] about to PICK from RR %p", this, - rr_policy_.get()); + if (!started_picking_) { + StartPickingLocked(); } - pick_done = PickFromRoundRobinPolicyLocked(false /* force_async */, pp); - } - } else { // rr_policy_ == NULL - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, - "[grpclb %p] No RR policy. Adding to grpclb's pending picks", - this); - } - AddPendingPick(pp); - if (!started_picking_) { - StartPickingLocked(); + pick_done = false; } - pick_done = false; } return pick_done; } -void GrpcLb::PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) { - if (rr_policy_ != nullptr) { - rr_policy_->PingOneLocked(on_initiate, on_ack); - } else { - AddPendingPing(on_initiate, on_ack); - if (!started_picking_) { - StartPickingLocked(); - } - } -} - void GrpcLb::FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) { // delegate to the RoundRobin to fill the children subchannels. rr_policy_->FillChildRefsForChannelz(child_subchannels, child_channels); - mu_guard guard(&lb_channel_mu_); + MutexLock lock(&lb_channel_mu_); if (lb_channel_ != nullptr) { grpc_core::channelz::ChannelNode* channel_node = grpc_channel_get_channelz_node(lb_channel_); @@ -1598,18 +1569,6 @@ void GrpcLb::AddPendingPick(PendingPick* pp) { pending_picks_ = pp; } -// -// PendingPing -// - -void GrpcLb::AddPendingPing(grpc_closure* on_initiate, grpc_closure* on_ack) { - PendingPing* pping = New(); - pping->on_initiate = on_initiate; - pping->on_ack = on_ack; - pping->next = pending_pings_; - pending_pings_ = pping; -} - // // code for interacting with the RR policy // @@ -1619,7 +1578,8 @@ void GrpcLb::AddPendingPing(grpc_closure* on_initiate, grpc_closure* on_ack) { // cleanups this callback would otherwise be responsible for. // If \a force_async is true, then we will manually schedule the // completion callback even if the pick is available immediately. -bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp) { +bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp, + grpc_error** error) { // Check for drops if we are not using fallback backend addresses. if (serverlist_ != nullptr) { // Look at the index into the serverlist to see if we should drop this call. @@ -1653,11 +1613,12 @@ bool GrpcLb::PickFromRoundRobinPolicyLocked(bool force_async, PendingPick* pp) { GPR_ASSERT(pp->pick->user_data == nullptr); pp->pick->user_data = (void**)&pp->lb_token; // Pick via the RR policy. - bool pick_done = rr_policy_->PickLocked(pp->pick); + bool pick_done = rr_policy_->PickLocked(pp->pick, error); if (pick_done) { PendingPickSetMetadataAndContext(pp); if (force_async) { - GRPC_CLOSURE_SCHED(pp->original_on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pp->original_on_complete, *error); + *error = GRPC_ERROR_NONE; pick_done = false; } Delete(pp); @@ -1709,18 +1670,8 @@ void GrpcLb::CreateRoundRobinPolicyLocked(const Args& args) { "[grpclb %p] Pending pick about to (async) PICK from RR %p", this, rr_policy_.get()); } - PickFromRoundRobinPolicyLocked(true /* force_async */, pp); - } - // Send pending pings to RR policy. - PendingPing* pping; - while ((pping = pending_pings_)) { - pending_pings_ = pping->next; - if (grpc_lb_glb_trace.enabled()) { - gpr_log(GPR_INFO, "[grpclb %p] Pending ping about to PING from RR %p", - this, rr_policy_.get()); - } - rr_policy_->PingOneLocked(pping->on_initiate, pping->on_ack); - Delete(pping); + grpc_error* error = GRPC_ERROR_NONE; + PickFromRoundRobinPolicyLocked(true /* force_async */, pp, &error); } } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 023281db974..9120abfa3c4 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -27,6 +27,7 @@ #include "src/core/ext/filters/client_channel/subchannel.h" #include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/transport/connectivity_state.h" @@ -46,7 +47,7 @@ class PickFirst : public LoadBalancingPolicy { explicit PickFirst(const Args& args); void UpdateLocked(const grpc_channel_args& args) override; - bool PickLocked(PickState* pick) override; + bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq, @@ -56,8 +57,8 @@ class PickFirst : public LoadBalancingPolicy { grpc_connectivity_state CheckConnectivityLocked( grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; - void PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -80,6 +81,11 @@ class PickFirst : public LoadBalancingPolicy { void ProcessConnectivityChangeLocked( grpc_connectivity_state connectivity_state, grpc_error* error) override; + + // Processes the connectivity change to READY for an unselected subchannel. + void ProcessUnselectedReadyLocked(); + + void CheckConnectivityStateAndStartWatchingLocked(); }; class PickFirstSubchannelList @@ -173,9 +179,10 @@ void PickFirst::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { PickState* pick; while ((pick = pending_picks_) != nullptr) { pending_picks_ = pick->next; - if (new_policy->PickLocked(pick)) { + grpc_error* error = GRPC_ERROR_NONE; + if (new_policy->PickLocked(pick, &error)) { // Synchronous return, schedule closure. - GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pick->on_complete, error); } } } @@ -246,7 +253,8 @@ void PickFirst::StartPickingLocked() { if (subchannel_list_ != nullptr) { for (size_t i = 0; i < subchannel_list_->num_subchannels(); ++i) { if (subchannel_list_->subchannel(i)->subchannel() != nullptr) { - subchannel_list_->subchannel(i)->StartConnectivityWatchLocked(); + subchannel_list_->subchannel(i) + ->CheckConnectivityStateAndStartWatchingLocked(); break; } } @@ -259,18 +267,30 @@ void PickFirst::ExitIdleLocked() { } } -bool PickFirst::PickLocked(PickState* pick) { +void PickFirst::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + +bool PickFirst::PickLocked(PickState* pick, grpc_error** error) { // If we have a selected subchannel already, return synchronously. if (selected_ != nullptr) { pick->connected_subchannel = selected_->connected_subchannel()->Ref(); return true; } // No subchannel selected yet, so handle asynchronously. - if (!started_picking_) { - StartPickingLocked(); + if (pick->on_complete == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No pick result available but synchronous result required."); + return true; } pick->next = pending_picks_; pending_picks_ = pick; + if (!started_picking_) { + StartPickingLocked(); + } return false; } @@ -293,20 +313,9 @@ void PickFirst::NotifyOnStateChangeLocked(grpc_connectivity_state* current, notify); } -void PickFirst::PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) { - if (selected_ != nullptr) { - selected_->connected_subchannel()->Ping(on_initiate, on_ack); - } else { - GRPC_CLOSURE_SCHED(on_initiate, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Not connected")); - GRPC_CLOSURE_SCHED(on_ack, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Not connected")); - } -} - void PickFirst::FillChildRefsForChannelz( ChildRefsList* child_subchannels_to_fill, ChildRefsList* ignored) { - mu_guard guard(&child_refs_mu_); + MutexLock lock(&child_refs_mu_); for (size_t i = 0; i < child_subchannels_.size(); ++i) { // TODO(ncteisen): implement a de dup loop that is not O(n^2). Might // have to implement lightweight set. For now, we don't care about @@ -333,7 +342,7 @@ void PickFirst::UpdateChildRefsLocked() { latest_pending_subchannel_list_->PopulateChildRefsList(&cs); } // atomically update the data that channelz will actually be looking at. - mu_guard guard(&child_refs_mu_); + MutexLock lock(&child_refs_mu_); child_subchannels_ = std::move(cs); } @@ -384,7 +393,8 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { // If we've started picking, start trying to connect to the first // subchannel in the new list. if (started_picking_) { - subchannel_list_->subchannel(0)->StartConnectivityWatchLocked(); + subchannel_list_->subchannel(0) + ->CheckConnectivityStateAndStartWatchingLocked(); } } else { // We do have a selected subchannel. @@ -438,7 +448,7 @@ void PickFirst::UpdateLocked(const grpc_channel_args& args) { // subchannel in the new list. if (started_picking_) { latest_pending_subchannel_list_->subchannel(0) - ->StartConnectivityWatchLocked(); + ->CheckConnectivityStateAndStartWatchingLocked(); } } } @@ -451,6 +461,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // latest pending subchannel lists. GPR_ASSERT(subchannel_list() == p->subchannel_list_.get() || subchannel_list() == p->latest_pending_subchannel_list_.get()); + GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN); // Handle updates for the currently selected subchannel. if (p->selected_ == this) { if (grpc_lb_pick_first_trace.enabled()) { @@ -480,14 +491,12 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( "update"), "selected_not_ready+switch_to_update"); } else { - // TODO(juanlishen): we re-resolve when the selected subchannel goes to - // TRANSIENT_FAILURE because we used to shut down in this case before - // re-resolution is introduced. But we need to investigate whether we - // really want to take any action instead of waiting for the selected - // subchannel reconnecting. - GPR_ASSERT(connectivity_state != GRPC_CHANNEL_SHUTDOWN); if (connectivity_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - // If the selected channel goes bad, request a re-resolution. + // If the selected subchannel goes bad, request a re-resolution. We also + // set the channel state to IDLE and reset started_picking_. The reason + // is that if the new state is TRANSIENT_FAILURE due to a GOAWAY + // reception we don't want to connect to the re-resolved backends until + // we leave the IDLE state. grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_IDLE, GRPC_ERROR_NONE, "selected_changed+reresolve"); @@ -518,41 +527,7 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // select in place of the current one. switch (connectivity_state) { case GRPC_CHANNEL_READY: { - // Case 2. Promote p->latest_pending_subchannel_list_ to - // p->subchannel_list_. - if (subchannel_list() == p->latest_pending_subchannel_list_.get()) { - if (grpc_lb_pick_first_trace.enabled()) { - gpr_log(GPR_INFO, - "Pick First %p promoting pending subchannel list %p to " - "replace %p", - p, p->latest_pending_subchannel_list_.get(), - p->subchannel_list_.get()); - } - p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); - } - // Cases 1 and 2. - grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_READY, - GRPC_ERROR_NONE, "connecting_ready"); - p->selected_ = this; - if (grpc_lb_pick_first_trace.enabled()) { - gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p, - subchannel()); - } - // Drop all other subchannels, since we are now connected. - p->DestroyUnselectedSubchannelsLocked(); - // Update any calls that were waiting for a pick. - PickState* pick; - while ((pick = p->pending_picks_)) { - p->pending_picks_ = pick->next; - pick->connected_subchannel = - p->selected_->connected_subchannel()->Ref(); - if (grpc_lb_pick_first_trace.enabled()) { - gpr_log(GPR_INFO, - "Servicing pending pick with selected subchannel %p", - p->selected_->subchannel()); - } - GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); - } + ProcessUnselectedReadyLocked(); // Renew notification. RenewConnectivityWatchLocked(); break; @@ -568,11 +543,12 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( // Case 1: Only set state to TRANSIENT_FAILURE if we've tried // all subchannels. if (sd->Index() == 0 && subchannel_list() == p->subchannel_list_.get()) { + p->TryReresolutionLocked(&grpc_lb_pick_first_trace, GRPC_ERROR_NONE); grpc_connectivity_state_set( &p->state_tracker_, GRPC_CHANNEL_TRANSIENT_FAILURE, - GRPC_ERROR_REF(error), "connecting_transient_failure"); + GRPC_ERROR_REF(error), "exhausted_subchannels"); } - sd->StartConnectivityWatchLocked(); + sd->CheckConnectivityStateAndStartWatchingLocked(); break; } case GRPC_CHANNEL_CONNECTING: @@ -593,6 +569,67 @@ void PickFirst::PickFirstSubchannelData::ProcessConnectivityChangeLocked( GRPC_ERROR_UNREF(error); } +void PickFirst::PickFirstSubchannelData::ProcessUnselectedReadyLocked() { + PickFirst* p = static_cast(subchannel_list()->policy()); + // If we get here, there are two possible cases: + // 1. We do not currently have a selected subchannel, and the update is + // for a subchannel in p->subchannel_list_ that we're trying to + // connect to. The goal here is to find a subchannel that we can + // select. + // 2. We do currently have a selected subchannel, and the update is + // for a subchannel in p->latest_pending_subchannel_list_. The + // goal here is to find a subchannel from the update that we can + // select in place of the current one. + GPR_ASSERT(subchannel_list() == p->subchannel_list_.get() || + subchannel_list() == p->latest_pending_subchannel_list_.get()); + // Case 2. Promote p->latest_pending_subchannel_list_ to p->subchannel_list_. + if (subchannel_list() == p->latest_pending_subchannel_list_.get()) { + if (grpc_lb_pick_first_trace.enabled()) { + gpr_log(GPR_INFO, + "Pick First %p promoting pending subchannel list %p to " + "replace %p", + p, p->latest_pending_subchannel_list_.get(), + p->subchannel_list_.get()); + } + p->subchannel_list_ = std::move(p->latest_pending_subchannel_list_); + } + // Cases 1 and 2. + grpc_connectivity_state_set(&p->state_tracker_, GRPC_CHANNEL_READY, + GRPC_ERROR_NONE, "subchannel_ready"); + p->selected_ = this; + if (grpc_lb_pick_first_trace.enabled()) { + gpr_log(GPR_INFO, "Pick First %p selected subchannel %p", p, subchannel()); + } + // Drop all other subchannels, since we are now connected. + p->DestroyUnselectedSubchannelsLocked(); + // Update any calls that were waiting for a pick. + PickState* pick; + while ((pick = p->pending_picks_)) { + p->pending_picks_ = pick->next; + pick->connected_subchannel = p->selected_->connected_subchannel()->Ref(); + if (grpc_lb_pick_first_trace.enabled()) { + gpr_log(GPR_INFO, "Servicing pending pick with selected subchannel %p", + p->selected_->subchannel()); + } + GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + } +} + +void PickFirst::PickFirstSubchannelData:: + CheckConnectivityStateAndStartWatchingLocked() { + PickFirst* p = static_cast(subchannel_list()->policy()); + grpc_error* error = GRPC_ERROR_NONE; + if (p->selected_ != this && + CheckConnectivityStateLocked(&error) == GRPC_CHANNEL_READY) { + // We must process the READY subchannel before we start watching it. + // Otherwise, we won't know it's READY because we will be waiting for its + // connectivity state to change from READY. + ProcessUnselectedReadyLocked(); + } + GRPC_ERROR_UNREF(error); + StartConnectivityWatchLocked(); +} + // // factory // diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index fc56a4961f5..c730b3bd2b1 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -36,6 +36,7 @@ #include "src/core/ext/filters/client_channel/subchannel_index.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -57,7 +58,7 @@ class RoundRobin : public LoadBalancingPolicy { explicit RoundRobin(const Args& args); void UpdateLocked(const grpc_channel_args& args) override; - bool PickLocked(PickState* pick) override; + bool PickLocked(PickState* pick, grpc_error** error) override; void CancelPickLocked(PickState* pick, grpc_error* error) override; void CancelMatchingPicksLocked(uint32_t initial_metadata_flags_mask, uint32_t initial_metadata_flags_eq, @@ -67,8 +68,8 @@ class RoundRobin : public LoadBalancingPolicy { grpc_connectivity_state CheckConnectivityLocked( grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; - void PingOneLocked(grpc_closure* on_initiate, grpc_closure* on_ack) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -253,9 +254,10 @@ void RoundRobin::HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) { PickState* pick; while ((pick = pending_picks_) != nullptr) { pending_picks_ = pick->next; - if (new_policy->PickLocked(pick)) { + grpc_error* error = GRPC_ERROR_NONE; + if (new_policy->PickLocked(pick, &error)) { // Synchronous return, schedule closure. - GRPC_CLOSURE_SCHED(pick->on_complete, GRPC_ERROR_NONE); + GRPC_CLOSURE_SCHED(pick->on_complete, error); } } } @@ -333,6 +335,13 @@ void RoundRobin::ExitIdleLocked() { } } +void RoundRobin::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool RoundRobin::DoPickLocked(PickState* pick) { const size_t next_ready_index = subchannel_list_->GetNextReadySubchannelIndexLocked(); @@ -368,7 +377,7 @@ void RoundRobin::DrainPendingPicksLocked() { } } -bool RoundRobin::PickLocked(PickState* pick) { +bool RoundRobin::PickLocked(PickState* pick, grpc_error** error) { if (grpc_lb_round_robin_trace.enabled()) { gpr_log(GPR_INFO, "[RR %p] Trying to pick (shutdown: %d)", this, shutdown_); } @@ -376,6 +385,11 @@ bool RoundRobin::PickLocked(PickState* pick) { if (subchannel_list_ != nullptr) { if (DoPickLocked(pick)) return true; } + if (pick->on_complete == nullptr) { + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No pick result available but synchronous result required."); + return true; + } /* no pick currently available. Save for later in list of pending picks */ pick->next = pending_picks_; pending_picks_ = pick; @@ -387,7 +401,7 @@ bool RoundRobin::PickLocked(PickState* pick) { void RoundRobin::FillChildRefsForChannelz( ChildRefsList* child_subchannels_to_fill, ChildRefsList* ignored) { - mu_guard guard(&child_refs_mu_); + MutexLock lock(&child_refs_mu_); for (size_t i = 0; i < child_subchannels_.size(); ++i) { // TODO(ncteisen): implement a de dup loop that is not O(n^2). Might // have to implement lightweight set. For now, we don't care about @@ -414,7 +428,7 @@ void RoundRobin::UpdateChildRefsLocked() { latest_pending_subchannel_list_->PopulateChildRefsList(&cs); } // atomically update the data that channelz will actually be looking at. - mu_guard guard(&child_refs_mu_); + MutexLock lock(&child_refs_mu_); child_subchannels_ = std::move(cs); } @@ -647,22 +661,6 @@ void RoundRobin::NotifyOnStateChangeLocked(grpc_connectivity_state* current, notify); } -void RoundRobin::PingOneLocked(grpc_closure* on_initiate, - grpc_closure* on_ack) { - const size_t next_ready_index = - subchannel_list_->GetNextReadySubchannelIndexLocked(); - if (next_ready_index < subchannel_list_->num_subchannels()) { - RoundRobinSubchannelData* selected = - subchannel_list_->subchannel(next_ready_index); - selected->connected_subchannel()->Ping(on_initiate, on_ack); - } else { - GRPC_CLOSURE_SCHED(on_initiate, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Round Robin not connected")); - GRPC_CLOSURE_SCHED(on_ack, GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Round Robin not connected")); - } -} - void RoundRobin::UpdateLocked(const grpc_channel_args& args) { const grpc_arg* arg = grpc_channel_args_find(&args, GRPC_ARG_LB_ADDRESSES); AutoChildRefsUpdater guard(this); diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 018ac3bb868..0fa2f04e73b 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -107,6 +107,11 @@ class SubchannelData { // being unreffed. virtual void UnrefSubchannelLocked(const char* reason); + // Resets the connection backoff. + // TODO(roth): This method should go away when we move the backoff + // code out of the subchannel and into the LB policies. + void ResetBackoffLocked(); + // Starts watching the connectivity state of the subchannel. // ProcessConnectivityChangeLocked() will be called when the // connectivity state changes. @@ -206,6 +211,11 @@ class SubchannelList LoadBalancingPolicy* policy() const { return policy_; } TraceFlag* tracer() const { return tracer_; } + // Resets connection backoff of all subchannels. + // TODO(roth): We will probably need to rethink this as part of moving + // the backoff code out of subchannels and into LB policies. + void ResetBackoffLocked(); + // Note: Caller must ensure that this is invoked inside of the combiner. void Orphan() override { ShutdownLocked(); @@ -298,6 +308,14 @@ void SubchannelData:: } } +template +void SubchannelData::ResetBackoffLocked() { + if (subchannel_ != nullptr) { + grpc_subchannel_reset_backoff(subchannel_); + } +} + template void SubchannelData::StartConnectivityWatchLocked() { @@ -544,6 +562,15 @@ void SubchannelList::ShutdownLocked() { } } +template +void SubchannelList::ResetBackoffLocked() { + for (size_t i = 0; i < subchannels_.size(); i++) { + SubchannelDataType* sd = &subchannels_[i]; + sd->ResetBackoffLocked(); + } +} + } // namespace grpc_core #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_SUBCHANNEL_LIST_H */ diff --git a/src/core/ext/filters/client_channel/resolver.h b/src/core/ext/filters/client_channel/resolver.h index c7e37e4468e..48f2e890952 100644 --- a/src/core/ext/filters/client_channel/resolver.h +++ b/src/core/ext/filters/client_channel/resolver.h @@ -94,6 +94,14 @@ class Resolver : public InternallyRefCountedWithTracing { /// throw away unselected subchannels. virtual void RequestReresolutionLocked() GRPC_ABSTRACT; + /// Resets the re-resolution backoff, if any. + /// This needs to be implemented only by pull-based implementations; + /// for push-based implementations, it will be a no-op. + /// TODO(roth): Pull the backoff code out of resolver and into + /// client_channel, so that it can be shared across resolver + /// implementations. At that point, this method can go away. + virtual void ResetBackoffLocked() {} + void Orphan() override { // Invoke ShutdownAndUnrefLocked() inside of the combiner. GRPC_CLOSURE_SCHED( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index f4f6444c5fa..f2bb5f3c712 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -23,7 +23,6 @@ #include #include #include -#include #include #include @@ -67,6 +66,8 @@ class AresDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -142,8 +143,8 @@ AresDnsResolver::AresDnsResolver(const ResolverArgs& args) channel_args_ = grpc_channel_args_copy(args.args); const grpc_arg* arg = grpc_channel_args_find( channel_args_, GRPC_ARG_SERVICE_CONFIG_DISABLE_RESOLUTION); - request_service_config_ = !grpc_channel_arg_get_integer( - arg, (grpc_integer_options){false, false, true}); + grpc_integer_options integer_options = {false, false, true}; + request_service_config_ = !grpc_channel_arg_get_integer(arg, integer_options); arg = grpc_channel_args_find(channel_args_, GRPC_ARG_DNS_MIN_TIME_BETWEEN_RESOLUTIONS_MS); min_time_between_resolutions_ = @@ -188,6 +189,13 @@ void AresDnsResolver::RequestReresolutionLocked() { } } +void AresDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void AresDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc index c886795608f..fdbd07ebf51 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc @@ -18,11 +18,10 @@ #include #include "src/core/lib/iomgr/port.h" -#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) +#if GRPC_ARES == 1 && !defined(GRPC_UV) #include #include -#include #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" @@ -32,7 +31,6 @@ #include #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/lib/gpr/string.h" -#include "src/core/lib/iomgr/ev_posix.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/sockaddr_utils.h" @@ -76,6 +74,8 @@ struct grpc_ares_ev_driver { bool shutting_down; /** request object that's using this ev driver */ grpc_ares_request* request; + /** Owned by the ev_driver. Creates new GrpcPolledFd's */ + grpc_core::UniquePtr polled_fd_factory; }; static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver); @@ -95,7 +95,7 @@ static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) { GRPC_COMBINER_UNREF(ev_driver->combiner, "free ares event driver"); ares_destroy(ev_driver->channel); grpc_ares_complete_request_locked(ev_driver->request); - gpr_free(ev_driver); + grpc_core::Delete(ev_driver); } } @@ -120,13 +120,11 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, grpc_pollset_set* pollset_set, grpc_combiner* combiner, grpc_ares_request* request) { - *ev_driver = static_cast( - gpr_malloc(sizeof(grpc_ares_ev_driver))); + *ev_driver = grpc_core::New(); ares_options opts; memset(&opts, 0, sizeof(opts)); opts.flags |= ARES_FLAG_STAYOPEN; int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS); - grpc_core::ConfigureAresChannelLocked(&(*ev_driver)->channel); gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create_locked"); if (status != ARES_SUCCESS) { char* err_msg; @@ -144,6 +142,10 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, (*ev_driver)->working = false; (*ev_driver)->shutting_down = false; (*ev_driver)->request = request; + (*ev_driver)->polled_fd_factory = + grpc_core::NewGrpcPolledFdFactory((*ev_driver)->combiner); + (*ev_driver) + ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel); return GRPC_ERROR_NONE; } @@ -247,8 +249,9 @@ static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) { // Create a new fd_node if sock[i] is not in the fd_node list. if (fdn == nullptr) { fdn = static_cast(gpr_malloc(sizeof(fd_node))); - fdn->grpc_polled_fd = grpc_core::NewGrpcPolledFdLocked( - socks[i], ev_driver->pollset_set); + fdn->grpc_polled_fd = + ev_driver->polled_fd_factory->NewGrpcPolledFdLocked( + socks[i], ev_driver->pollset_set, ev_driver->combiner); gpr_log(GPR_DEBUG, "new fd: %s", fdn->grpc_polled_fd->GetName()); fdn->ev_driver = ev_driver; fdn->readable_registered = false; @@ -314,4 +317,4 @@ void grpc_ares_ev_driver_start_locked(grpc_ares_ev_driver* ev_driver) { } } -#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */ +#endif /* GRPC_ARES == 1 && !defined(GRPC_UV) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h index 2c9db710116..671c537fe72 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h @@ -81,10 +81,24 @@ class GrpcPolledFd { GRPC_ABSTRACT_BASE_CLASS }; -/* Creates a new wrapped fd for the current platform */ -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set); -void ConfigureAresChannelLocked(ares_channel* channel); +/* A GrpcPolledFdFactory is 1-to-1 with and owned by the + * ares event driver. It knows how to create GrpcPolledFd's + * for the current platform, and the ares driver uses it for all of + * its fd's. */ +class GrpcPolledFdFactory { + public: + virtual ~GrpcPolledFdFactory() {} + /* Creates a new wrapped fd for the current platform */ + virtual GrpcPolledFd* NewGrpcPolledFdLocked( + ares_socket_t as, grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) GRPC_ABSTRACT; + /* Optionally configures the ares channel after creation */ + virtual void ConfigureAresChannelLocked(ares_channel channel) GRPC_ABSTRACT; + + GRPC_ABSTRACT_BASE_CLASS +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner); } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc index fffe9eda8e5..aa58e1aaf50 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc @@ -86,12 +86,20 @@ class GrpcPolledFdPosix : public GrpcPolledFd { grpc_pollset_set* driver_pollset_set_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return grpc_core::New(as, driver_pollset_set); -} +class GrpcPolledFdFactoryPosix : public GrpcPolledFdFactory { + public: + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + return New(as, driver_pollset_set); + } -void ConfigureAresChannelLocked(ares_channel* channel) {} + void ConfigureAresChannelLocked(ares_channel channel) override {} +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr(New()); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc new file mode 100644 index 00000000000..02121aa0ab4 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc @@ -0,0 +1,537 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +#include + +#include "src/core/lib/iomgr/port.h" +#if GRPC_ARES == 1 && defined(GPR_WINDOWS) + +#include + +#include +#include +#include +#include +#include +#include +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#include "src/core/lib/slice/slice_internal.h" + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" + +/* TODO(apolcyn): remove this hack after fixing upstream. + * Our grpc/c-ares code on Windows uses the ares_set_socket_functions API, + * which uses "struct iovec" type, which on Windows is defined inside of + * a c-ares header that is not public. + * See https://github.com/c-ares/c-ares/issues/206. */ +struct iovec { + void* iov_base; + size_t iov_len; +}; + +namespace grpc_core { + +/* c-ares creates its own sockets and is meant to read them when readable and + * write them when writeable. To fit this socket usage model into the grpc + * windows poller (which gives notifications when attempted reads and writes are + * actually fulfilled rather than possible), this GrpcPolledFdWindows class + * takes advantage of the ares_set_socket_functions API and acts as a virtual + * socket. It holds its own read and write buffers which are written to and read + * from c-ares and are used with the grpc windows poller, and it, e.g., + * manufactures virtual socket error codes when it e.g. needs to tell the c-ares + * library to wait for an async read. */ +class GrpcPolledFdWindows : public GrpcPolledFd { + public: + enum WriteState { + WRITE_IDLE, + WRITE_REQUESTED, + WRITE_PENDING, + WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY, + }; + + GrpcPolledFdWindows(ares_socket_t as, grpc_combiner* combiner) + : read_buf_(grpc_empty_slice()), + write_buf_(grpc_empty_slice()), + write_state_(WRITE_IDLE), + gotten_into_driver_list_(false) { + gpr_asprintf(&name_, "c-ares socket: %" PRIdPTR, as); + winsocket_ = grpc_winsocket_create(as, name_); + combiner_ = GRPC_COMBINER_REF(combiner, name_); + GRPC_CLOSURE_INIT(&outer_read_closure_, + &GrpcPolledFdWindows::OnIocpReadable, this, + grpc_combiner_scheduler(combiner_)); + GRPC_CLOSURE_INIT(&outer_write_closure_, + &GrpcPolledFdWindows::OnIocpWriteable, this, + grpc_combiner_scheduler(combiner_)); + } + + ~GrpcPolledFdWindows() { + GRPC_COMBINER_UNREF(combiner_, name_); + grpc_slice_unref_internal(read_buf_); + grpc_slice_unref_internal(write_buf_); + GPR_ASSERT(read_closure_ == nullptr); + GPR_ASSERT(write_closure_ == nullptr); + grpc_winsocket_destroy(winsocket_); + gpr_free(name_); + } + + void ScheduleAndNullReadClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(read_closure_, error); + read_closure_ = nullptr; + } + + void ScheduleAndNullWriteClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(write_closure_, error); + write_closure_ = nullptr; + } + + void RegisterForOnReadableLocked(grpc_closure* read_closure) override { + GPR_ASSERT(read_closure_ == nullptr); + read_closure_ = read_closure; + GPR_ASSERT(GRPC_SLICE_LENGTH(read_buf_) == 0); + grpc_slice_unref_internal(read_buf_); + read_buf_ = GRPC_SLICE_MALLOC(4192); + WSABUF buffer; + buffer.buf = (char*)GRPC_SLICE_START_PTR(read_buf_); + buffer.len = GRPC_SLICE_LENGTH(read_buf_); + memset(&winsocket_->read_info.overlapped, 0, sizeof(OVERLAPPED)); + recv_from_source_addr_len_ = sizeof(recv_from_source_addr_); + DWORD flags = 0; + if (WSARecvFrom(grpc_winsocket_wrapped_socket(winsocket_), &buffer, 1, + nullptr, &flags, (sockaddr*)recv_from_source_addr_, + &recv_from_source_addr_len_, + &winsocket_->read_info.overlapped, nullptr)) { + char* msg = gpr_format_message(WSAGetLastError()); + grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + GRPC_CARES_TRACE_LOG( + "RegisterForOnReadableLocked: WSARecvFrom error:|%s|. fd:|%s|", msg, + GetName()); + gpr_free(msg); + if (WSAGetLastError() != WSA_IO_PENDING) { + ScheduleAndNullReadClosure(error); + return; + } + } + grpc_socket_notify_on_read(winsocket_, &outer_read_closure_); + } + + void RegisterForOnWriteableLocked(grpc_closure* write_closure) override { + GRPC_CARES_TRACE_LOG( + "RegisterForOnWriteableLocked. fd:|%s|. Current write state: %d", + GetName(), write_state_); + GPR_ASSERT(write_closure_ == nullptr); + write_closure_ = write_closure; + switch (write_state_) { + case WRITE_IDLE: + ScheduleAndNullWriteClosure(GRPC_ERROR_NONE); + break; + case WRITE_REQUESTED: + write_state_ = WRITE_PENDING; + SendWriteBuf(nullptr, &winsocket_->write_info.overlapped); + grpc_socket_notify_on_write(winsocket_, &outer_write_closure_); + break; + case WRITE_PENDING: + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + abort(); + } + } + + bool IsFdStillReadableLocked() override { + return GRPC_SLICE_LENGTH(read_buf_) > 0; + } + + void ShutdownLocked(grpc_error* error) override { + grpc_winsocket_shutdown(winsocket_); + } + + ares_socket_t GetWrappedAresSocketLocked() override { + return grpc_winsocket_wrapped_socket(winsocket_); + } + + const char* GetName() override { return name_; } + + ares_ssize_t RecvFrom(void* data, ares_socket_t data_len, int flags, + struct sockaddr* from, ares_socklen_t* from_len) { + GRPC_CARES_TRACE_LOG( + "RecvFrom called on fd:|%s|. Current read buf length:|%d|", GetName(), + GRPC_SLICE_LENGTH(read_buf_)); + if (GRPC_SLICE_LENGTH(read_buf_) == 0) { + WSASetLastError(WSAEWOULDBLOCK); + return -1; + } + ares_ssize_t bytes_read = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(read_buf_) && i < data_len; i++) { + ((char*)data)[i] = GRPC_SLICE_START_PTR(read_buf_)[i]; + bytes_read++; + } + read_buf_ = grpc_slice_sub_no_ref(read_buf_, bytes_read, + GRPC_SLICE_LENGTH(read_buf_)); + /* c-ares overloads this recv_from virtual socket function to receive + * data on both UDP and TCP sockets, and from is nullptr for TCP. */ + if (from != nullptr) { + GPR_ASSERT(*from_len <= recv_from_source_addr_len_); + memcpy(from, &recv_from_source_addr_, recv_from_source_addr_len_); + *from_len = recv_from_source_addr_len_; + } + return bytes_read; + } + + grpc_slice FlattenIovec(const struct iovec* iov, int iov_count) { + int total = 0; + for (int i = 0; i < iov_count; i++) { + total += iov[i].iov_len; + } + grpc_slice out = GRPC_SLICE_MALLOC(total); + size_t cur = 0; + for (int i = 0; i < iov_count; i++) { + for (int k = 0; k < iov[i].iov_len; k++) { + GRPC_SLICE_START_PTR(out)[cur++] = ((char*)iov[i].iov_base)[k]; + } + } + return out; + } + + int SendWriteBuf(LPDWORD bytes_sent_ptr, LPWSAOVERLAPPED overlapped) { + WSABUF buf; + buf.len = GRPC_SLICE_LENGTH(write_buf_); + buf.buf = (char*)GRPC_SLICE_START_PTR(write_buf_); + DWORD flags = 0; + int out = WSASend(grpc_winsocket_wrapped_socket(winsocket_), &buf, 1, + bytes_sent_ptr, flags, overlapped, nullptr); + GRPC_CARES_TRACE_LOG( + "WSASend: name:%s. buf len:%d. bytes sent: %d. overlapped %p. return " + "val: %d", + GetName(), buf.len, *bytes_sent_ptr, overlapped, out); + return out; + } + + ares_ssize_t TrySendWriteBufSyncNonBlocking() { + GPR_ASSERT(write_state_ == WRITE_IDLE); + ares_ssize_t total_sent; + DWORD bytes_sent = 0; + if (SendWriteBuf(&bytes_sent, nullptr) != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG( + "TrySendWriteBufSyncNonBlocking: SendWriteBuf error:|%s|. fd:|%s|", + msg, GetName()); + gpr_free(msg); + if (WSAGetLastError() == WSA_IO_PENDING) { + WSASetLastError(WSAEWOULDBLOCK); + write_state_ = WRITE_REQUESTED; + } + } + write_buf_ = grpc_slice_sub_no_ref(write_buf_, bytes_sent, + GRPC_SLICE_LENGTH(write_buf_)); + return bytes_sent; + } + + ares_ssize_t SendV(const struct iovec* iov, int iov_count) { + GRPC_CARES_TRACE_LOG("SendV called on fd:|%s|. Current write state: %d", + GetName(), write_state_); + switch (write_state_) { + case WRITE_IDLE: + GPR_ASSERT(GRPC_SLICE_LENGTH(write_buf_) == 0); + grpc_slice_unref_internal(write_buf_); + write_buf_ = FlattenIovec(iov, iov_count); + return TrySendWriteBufSyncNonBlocking(); + case WRITE_REQUESTED: + case WRITE_PENDING: + WSASetLastError(WSAEWOULDBLOCK); + return -1; + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + grpc_slice currently_attempted = FlattenIovec(iov, iov_count); + GPR_ASSERT(GRPC_SLICE_LENGTH(currently_attempted) >= + GRPC_SLICE_LENGTH(write_buf_)); + ares_ssize_t total_sent = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(write_buf_); i++) { + GPR_ASSERT(GRPC_SLICE_START_PTR(currently_attempted)[i] == + GRPC_SLICE_START_PTR(write_buf_)[i]); + total_sent++; + } + grpc_slice_unref_internal(write_buf_); + write_buf_ = + grpc_slice_sub_no_ref(currently_attempted, total_sent, + GRPC_SLICE_LENGTH(currently_attempted)); + write_state_ = WRITE_IDLE; + total_sent += TrySendWriteBufSyncNonBlocking(); + return total_sent; + } + abort(); + } + + int Connect(const struct sockaddr* target, ares_socklen_t target_len) { + SOCKET s = grpc_winsocket_wrapped_socket(winsocket_); + GRPC_CARES_TRACE_LOG("Connect: fd:|%s|", GetName()); + int out = + WSAConnect(s, target, target_len, nullptr, nullptr, nullptr, nullptr); + if (out != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG("Connect error code:|%d|, msg:|%s|. fd:|%s|", + WSAGetLastError(), msg, GetName()); + gpr_free(msg); + // c-ares expects a posix-style connect API + out = -1; + } + return out; + } + + static void OnIocpReadable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpReadableInner(error); + } + + void OnIocpReadableInner(grpc_error* error) { + if (error == GRPC_ERROR_NONE) { + if (winsocket_->read_info.wsa_error != 0) { + /* WSAEMSGSIZE would be due to receiving more data + * than our read buffer's fixed capacity. Assume that + * the connection is TCP and read the leftovers + * in subsequent c-ares reads. */ + if (winsocket_->read_info.wsa_error != WSAEMSGSIZE) { + GRPC_ERROR_UNREF(error); + char* msg = gpr_format_message(winsocket_->read_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpReadableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + } + if (error == GRPC_ERROR_NONE) { + read_buf_ = grpc_slice_sub_no_ref(read_buf_, 0, + winsocket_->read_info.bytes_transfered); + } else { + grpc_slice_unref_internal(read_buf_); + read_buf_ = grpc_empty_slice(); + } + GRPC_CARES_TRACE_LOG( + "OnIocpReadable finishing. read buf length now:|%d|. :fd:|%s|", + GRPC_SLICE_LENGTH(read_buf_), GetName()); + ScheduleAndNullReadClosure(error); + } + + static void OnIocpWriteable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpWriteableInner(error); + } + + void OnIocpWriteableInner(grpc_error* error) { + GRPC_CARES_TRACE_LOG("OnIocpWriteableInner. fd:|%s|", GetName()); + if (error == GRPC_ERROR_NONE) { + if (winsocket_->write_info.wsa_error != 0) { + char* msg = gpr_format_message(winsocket_->write_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpWriteableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + GRPC_ERROR_UNREF(error); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + GPR_ASSERT(write_state_ == WRITE_PENDING); + if (error == GRPC_ERROR_NONE) { + write_state_ = WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY; + write_buf_ = grpc_slice_sub_no_ref( + write_buf_, 0, winsocket_->write_info.bytes_transfered); + } else { + grpc_slice_unref_internal(write_buf_); + write_buf_ = grpc_empty_slice(); + } + ScheduleAndNullWriteClosure(error); + } + + bool gotten_into_driver_list() const { return gotten_into_driver_list_; } + void set_gotten_into_driver_list() { gotten_into_driver_list_ = true; } + + grpc_combiner* combiner_; + char recv_from_source_addr_[200]; + ares_socklen_t recv_from_source_addr_len_; + grpc_slice read_buf_; + grpc_slice write_buf_; + grpc_closure* read_closure_ = nullptr; + grpc_closure* write_closure_ = nullptr; + grpc_closure outer_read_closure_; + grpc_closure outer_write_closure_; + grpc_winsocket* winsocket_; + WriteState write_state_; + char* name_ = nullptr; + bool gotten_into_driver_list_; +}; + +struct SockToPolledFdEntry { + SockToPolledFdEntry(SOCKET s, GrpcPolledFdWindows* fd) + : socket(s), polled_fd(fd) {} + SOCKET socket; + GrpcPolledFdWindows* polled_fd; + SockToPolledFdEntry* next = nullptr; +}; + +/* A SockToPolledFdMap can make ares_socket_t types (SOCKET's on windows) + * to GrpcPolledFdWindow's, and is used to find the appropriate + * GrpcPolledFdWindows to handle a virtual socket call when c-ares makes that + * socket call on the ares_socket_t type. Instances are owned by and one-to-one + * with a GrpcPolledFdWindows factory and event driver */ +class SockToPolledFdMap { + public: + SockToPolledFdMap(grpc_combiner* combiner) { + combiner_ = GRPC_COMBINER_REF(combiner, "sock to polled fd map"); + } + + ~SockToPolledFdMap() { + GPR_ASSERT(head_ == nullptr); + GRPC_COMBINER_UNREF(combiner_, "sock to polled fd map"); + } + + void AddNewSocket(SOCKET s, GrpcPolledFdWindows* polled_fd) { + SockToPolledFdEntry* new_node = New(s, polled_fd); + new_node->next = head_; + head_ = new_node; + } + + GrpcPolledFdWindows* LookupPolledFd(SOCKET s) { + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + GPR_ASSERT(node->polled_fd != nullptr); + return node->polled_fd; + } + } + abort(); + } + + void RemoveEntry(SOCKET s) { + GPR_ASSERT(head_ != nullptr); + SockToPolledFdEntry** prev = &head_; + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + *prev = node->next; + Delete(node); + return; + } + prev = &node->next; + } + abort(); + } + + /* These virtual socket functions are called from within the c-ares + * library. These methods generally dispatch those socket calls to the + * appropriate methods. The virtual "socket" and "close" methods are + * special and instead create/add and remove/destroy GrpcPolledFdWindows + * objects. + */ + static ares_socket_t Socket(int af, int type, int protocol, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + SOCKET s = WSASocket(af, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (s == INVALID_SOCKET) { + return s; + } + grpc_tcp_set_non_block(s); + GrpcPolledFdWindows* polled_fd = + New(s, map->combiner_); + map->AddNewSocket(s, polled_fd); + return s; + } + + static int Connect(ares_socket_t as, const struct sockaddr* target, + ares_socklen_t target_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->Connect(target, target_len); + } + + static ares_ssize_t SendV(ares_socket_t as, const struct iovec* iov, + int iovec_count, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->SendV(iov, iovec_count); + } + + static ares_ssize_t RecvFrom(ares_socket_t as, void* data, size_t data_len, + int flags, struct sockaddr* from, + ares_socklen_t* from_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->RecvFrom(data, data_len, flags, from, from_len); + } + + static int CloseSocket(SOCKET s, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(s); + map->RemoveEntry(s); + // If a gRPC polled fd has not made it in to the driver's list yet, then + // the driver has not and will never see this socket. + if (!polled_fd->gotten_into_driver_list()) { + polled_fd->ShutdownLocked(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Shut down c-ares fd before without it ever having made it into the " + "driver's list")); + return 0; + } + return 0; + } + + private: + SockToPolledFdEntry* head_ = nullptr; + grpc_combiner* combiner_; +}; + +const struct ares_socket_functions custom_ares_sock_funcs = { + &SockToPolledFdMap::Socket /* socket */, + &SockToPolledFdMap::CloseSocket /* close */, + &SockToPolledFdMap::Connect /* connect */, + &SockToPolledFdMap::RecvFrom /* recvfrom */, + &SockToPolledFdMap::SendV /* sendv */, +}; + +class GrpcPolledFdFactoryWindows : public GrpcPolledFdFactory { + public: + GrpcPolledFdFactoryWindows(grpc_combiner* combiner) + : sock_to_polled_fd_map_(combiner) {} + + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + GrpcPolledFdWindows* polled_fd = sock_to_polled_fd_map_.LookupPolledFd(as); + // Set a flag so that the virtual socket "close" method knows it + // doesn't need to call ShutdownLocked, since now the driver will. + polled_fd->set_gotten_into_driver_list(); + return polled_fd; + } + + void ConfigureAresChannelLocked(ares_channel channel) override { + ares_set_socket_functions(channel, &custom_ares_sock_funcs, + &sock_to_polled_fd_map_); + } + + private: + SockToPolledFdMap sock_to_polled_fd_map_; +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr( + New(combiner)); +} + +} // namespace grpc_core + +#endif /* GRPC_ARES == 1 && defined(GPR_WINDOWS) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index 497ad998af1..485998f5e44 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -22,7 +22,6 @@ #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" #include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/socket_utils_posix.h" #include #include @@ -50,6 +49,8 @@ static gpr_mu g_init_mu; grpc_core::TraceFlag grpc_trace_cares_address_sorting(false, "cares_address_sorting"); +grpc_core::TraceFlag grpc_trace_cares_resolver(false, "cares_resolver"); + struct grpc_ares_request { /** indicates the DNS server to use, if specified */ struct ares_addr_port_node dns_server_addr; @@ -215,7 +216,7 @@ static void on_hostbyname_done_locked(void* arg, int status, int timeouts, memset(&addr, 0, addr_len); memcpy(&addr.sin6_addr, hostent->h_addr_list[i - prev_naddr], sizeof(struct in6_addr)); - addr.sin6_family = static_cast(hostent->h_addrtype); + addr.sin6_family = static_cast(hostent->h_addrtype); addr.sin6_port = hr->port; grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, @@ -236,7 +237,7 @@ static void on_hostbyname_done_locked(void* arg, int status, int timeouts, memset(&addr, 0, addr_len); memcpy(&addr.sin_addr, hostent->h_addr_list[i - prev_naddr], sizeof(struct in_addr)); - addr.sin_family = static_cast(hostent->h_addrtype); + addr.sin_family = static_cast(hostent->h_addrtype); addr.sin_port = hr->port; grpc_lb_addresses_set_address( *lb_addresses, i, &addr, addr_len, @@ -281,7 +282,7 @@ static void on_srv_query_done_locked(void* arg, int status, int timeouts, grpc_ares_ev_driver_get_channel_locked(r->ev_driver); for (struct ares_srv_reply* srv_it = reply; srv_it != nullptr; srv_it = srv_it->next) { - if (grpc_ipv6_loopback_available()) { + if (grpc_ares_query_ipv6()) { grpc_ares_hostbyname_request* hr = create_hostbyname_request_locked( r, srv_it->host, htons(srv_it->port), true /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, @@ -452,7 +453,7 @@ static grpc_ares_request* grpc_dns_lookup_ares_locked_impl( } } r->pending_queries = 1; - if (grpc_ipv6_loopback_available()) { + if (grpc_ares_query_ipv6()) { hr = create_hostbyname_request_locked(r, host, strhtons(port), false /* is_balancer */); ares_gethostbyname(*channel, hr->host, AF_INET6, on_hostbyname_done_locked, diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index ce26f5d524a..ca5779e1d79 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -28,6 +28,13 @@ extern grpc_core::TraceFlag grpc_trace_cares_address_sorting; +extern grpc_core::TraceFlag grpc_trace_cares_resolver; + +#define GRPC_CARES_TRACE_LOG(format, ...) \ + if (grpc_trace_cares_resolver.enabled()) { \ + gpr_log(GPR_DEBUG, "(c-ares resolver) " format, __VA_ARGS__); \ + } + typedef struct grpc_ares_request grpc_ares_request; /* Asynchronously resolve \a name. Use \a default_port if a port isn't @@ -70,6 +77,10 @@ void grpc_ares_cleanup(void); * and destroys the grpc_ares_request */ void grpc_ares_complete_request_locked(grpc_ares_request* request); +/* Indicates whether or not AAAA queries should be attempted. */ +/* E.g., return false if ipv6 is known to not be available. */ +bool grpc_ares_query_ipv6(); + /* Exposed only for testing */ void grpc_cares_wrapper_test_only_address_sorting_sort( grpc_lb_addresses* lb_addrs); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc new file mode 100644 index 00000000000..23c0fec74f3 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc @@ -0,0 +1,29 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/iomgr/port.h" +#if GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/lib/iomgr/socket_utils_posix.h" + +bool grpc_ares_query_ipv6() { return grpc_ipv6_loopback_available(); } + +#endif /* GRPC_ARES == 1 && defined(GRPC_POSIX_SOCKET_ARES_EV_DRIVER) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc new file mode 100644 index 00000000000..ee827e284e2 --- /dev/null +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc @@ -0,0 +1,29 @@ +/* + * + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/iomgr/port.h" +#if GRPC_ARES == 1 && defined(GPR_WINDOWS) + +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" +#include "src/core/lib/iomgr/socket_windows.h" + +bool grpc_ares_query_ipv6() { return grpc_ipv6_loopback_available(); } + +#endif /* GRPC_ARES == 1 && defined(GPR_WINDOWS) */ diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc index fae4c33a178..282caf215c7 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc @@ -58,6 +58,8 @@ class NativeDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -158,6 +160,13 @@ void NativeDnsResolver::RequestReresolutionLocked() { } } +void NativeDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void NativeDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 93df2aff700..0e40f42e189 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -132,6 +132,8 @@ struct grpc_subchannel { bool have_alarm; /** have we started the backoff loop */ bool backoff_begun; + // reset_backoff() was called while alarm was pending + bool deferred_reset_backoff; /** our alarm */ grpc_timer alarm; @@ -403,7 +405,7 @@ static void continue_connect_locked(grpc_subchannel* c) { args.deadline = std::max(c->next_attempt_deadline, min_deadline); args.channel_args = c->args; grpc_connectivity_state_set(&c->state_tracker, GRPC_CHANNEL_CONNECTING, - GRPC_ERROR_NONE, "state_change"); + GRPC_ERROR_NONE, "connecting"); grpc_connector_connect(c->connector, &args, &c->connecting_result, &c->on_connected); } @@ -440,6 +442,9 @@ static void on_alarm(void* arg, grpc_error* error) { if (c->disconnected) { error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected", &error, 1); + } else if (c->deferred_reset_backoff) { + c->deferred_reset_backoff = false; + error = GRPC_ERROR_NONE; } else { GRPC_ERROR_REF(error); } @@ -459,25 +464,20 @@ static void maybe_start_connecting_locked(grpc_subchannel* c) { /* Don't try to connect if we're already disconnected */ return; } - if (c->connecting) { /* Already connecting: don't restart */ return; } - if (c->connected_subchannel != nullptr) { /* Already connected: don't restart */ return; } - if (!grpc_connectivity_state_has_watchers(&c->state_tracker)) { /* Nobody is interested in connecting: so don't just yet */ return; } - c->connecting = true; GRPC_SUBCHANNEL_WEAK_REF(c, "connecting"); - if (!c->backoff_begun) { c->backoff_begun = true; continue_connect_locked(c); @@ -675,6 +675,19 @@ static void on_subchannel_connected(void* arg, grpc_error* error) { grpc_channel_args_destroy(delete_channel_args); } +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel) { + gpr_mu_lock(&subchannel->mu); + if (subchannel->have_alarm) { + subchannel->deferred_reset_backoff = true; + grpc_timer_cancel(&subchannel->alarm); + } else { + subchannel->backoff_begun = false; + subchannel->backoff->Reset(); + maybe_start_connecting_locked(subchannel); + } + gpr_mu_unlock(&subchannel->mu); +} + /* * grpc_subchannel_call implementation */ diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 9e53f7d5423..a135035d62a 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -145,6 +145,13 @@ grpc_subchannel_get_connected_subchannel(grpc_subchannel* c); const grpc_subchannel_key* grpc_subchannel_get_key( const grpc_subchannel* subchannel); +// Resets the connection backoff of the subchannel. +// TODO(roth): Move connection backoff out of subchannels and up into LB +// policy code (probably by adding a SubchannelGroup between +// SubchannelList and SubchannelData), at which point this method can +// go away. +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel); + /** continue processing a transport op */ void grpc_subchannel_call_process_op(grpc_subchannel_call* subchannel_call, grpc_transport_stream_op_batch* op); diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index bc6fa0d0eb0..36511fa6088 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -812,6 +812,12 @@ static void set_write_state(grpc_chttp2_transport* t, write_state_name(t->write_state), write_state_name(st), reason)); t->write_state = st; + /* If the state is being reset back to idle, it means a write was just + * finished. Make sure all the run_after_write closures are scheduled. + * + * This is also our chance to close the transport if the transport was marked + * to be closed after all writes finish (for example, if we received a go-away + * from peer while we had some pending writes) */ if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) { GRPC_CLOSURE_LIST_SCHED(&t->run_after_write); if (t->close_transport_on_writes_finished != nullptr) { @@ -899,6 +905,22 @@ void grpc_chttp2_initiate_write(grpc_chttp2_transport* t, grpc_chttp2_initiate_write_reason_string(reason)); t->is_first_write_in_batch = true; GRPC_CHTTP2_REF_TRANSPORT(t, "writing"); + /* Note that the 'write_action_begin_locked' closure is being scheduled + * on the 'finally_scheduler' of t->combiner. This means that + * 'write_action_begin_locked' is called only *after* all the other + * closures (some of which are potentially initiating more writes on the + * transport) are executed on the t->combiner. + * + * The reason for scheduling on finally_scheduler is to make sure we batch + * as many writes as possible. 'write_action_begin_locked' is the function + * that gathers all the relevant bytes (which are at various places in the + * grpc_chttp2_transport structure) and append them to 'outbuf' field in + * grpc_chttp2_transport thereby batching what would have been potentially + * multiple write operations. + * + * Also, 'write_action_begin_locked' only gathers the bytes into outbuf. + * It does not call the endpoint to write the bytes. That is done by the + * 'write_action' (which is scheduled by 'write_action_begin_locked') */ GRPC_CLOSURE_SCHED( GRPC_CLOSURE_INIT(&t->write_action_begin_locked, write_action_begin_locked, t, @@ -1010,6 +1032,8 @@ static void write_action(void* gt, grpc_error* error) { grpc_combiner_scheduler(t->combiner))); } +/* Callback from the grpc_endpoint after bytes have been written by calling + * sendmsg */ static void write_action_end_locked(void* tp, grpc_error* error) { GPR_TIMER_SCOPE("terminate_writing_with_lock", 0); grpc_chttp2_transport* t = static_cast(tp); diff --git a/src/core/ext/transport/chttp2/transport/flow_control.cc b/src/core/ext/transport/chttp2/transport/flow_control.cc index e89c363200e..53932bcb7f5 100644 --- a/src/core/ext/transport/chttp2/transport/flow_control.cc +++ b/src/core/ext/transport/chttp2/transport/flow_control.cc @@ -40,6 +40,7 @@ namespace chttp2 { namespace { static constexpr const int kTracePadding = 30; +static constexpr const uint32_t kMaxWindowUpdateSize = (1u << 31) - 1; static char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) { char* str; @@ -55,7 +56,7 @@ static char* fmt_int64_diff_str(int64_t old_val, int64_t new_val) { static char* fmt_uint32_diff_str(uint32_t old_val, uint32_t new_val) { char* str; - if (new_val > 0 && old_val != new_val) { + if (old_val != new_val) { gpr_asprintf(&str, "%" PRIu32 " -> %" PRIu32 "", old_val, new_val); } else { gpr_asprintf(&str, "%" PRIu32 "", old_val); @@ -98,10 +99,12 @@ void FlowControlTrace::Finish() { if (sfc_ != nullptr) { srw_str = fmt_int64_diff_str(remote_window_delta_ + remote_window, sfc_->remote_window_delta() + remote_window); - slw_str = fmt_int64_diff_str(local_window_delta_ + acked_local_window, - local_window_delta_ + acked_local_window); - saw_str = fmt_int64_diff_str(announced_window_delta_ + acked_local_window, - announced_window_delta_ + acked_local_window); + slw_str = + fmt_int64_diff_str(local_window_delta_ + acked_local_window, + sfc_->local_window_delta() + acked_local_window); + saw_str = + fmt_int64_diff_str(announced_window_delta_ + acked_local_window, + sfc_->announced_window_delta() + acked_local_window); } else { srw_str = gpr_leftpad("", ' ', kTracePadding); slw_str = gpr_leftpad("", ' ', kTracePadding); @@ -191,7 +194,7 @@ uint32_t TransportFlowControl::MaybeSendUpdate(bool writing_anyway) { if ((writing_anyway || announced_window_ <= target_announced_window / 2) && announced_window_ != target_announced_window) { const uint32_t announce = static_cast GPR_CLAMP( - target_announced_window - announced_window_, 0, UINT32_MAX); + target_announced_window - announced_window_, 0, kMaxWindowUpdateSize); announced_window_ += announce; return announce; } @@ -265,7 +268,7 @@ uint32_t StreamFlowControl::MaybeSendUpdate() { FlowControlTrace trace("s updt sent", tfc_, this); if (local_window_delta_ > announced_window_delta_) { uint32_t announce = static_cast GPR_CLAMP( - local_window_delta_ - announced_window_delta_, 0, UINT32_MAX); + local_window_delta_ - announced_window_delta_, 0, kMaxWindowUpdateSize); UpdateAnnouncedWindowDelta(tfc_, announce); return announce; } diff --git a/src/core/lib/channel/channelz_registry.cc b/src/core/lib/channel/channelz_registry.cc index 38496b3d78c..f79d2f0c177 100644 --- a/src/core/lib/channel/channelz_registry.cc +++ b/src/core/lib/channel/channelz_registry.cc @@ -23,6 +23,7 @@ #include "src/core/lib/channel/channelz_registry.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/gprpp/mutex_lock.h" #include #include @@ -53,7 +54,7 @@ ChannelzRegistry::ChannelzRegistry() { gpr_mu_init(&mu_); } ChannelzRegistry::~ChannelzRegistry() { gpr_mu_destroy(&mu_); } intptr_t ChannelzRegistry::InternalRegisterEntry(const RegistryEntry& entry) { - mu_guard guard(&mu_); + MutexLock lock(&mu_); entities_.push_back(entry); intptr_t uuid = entities_.size(); return uuid; @@ -61,7 +62,7 @@ intptr_t ChannelzRegistry::InternalRegisterEntry(const RegistryEntry& entry) { void ChannelzRegistry::InternalUnregisterEntry(intptr_t uuid, EntityType type) { GPR_ASSERT(uuid >= 1); - mu_guard guard(&mu_); + MutexLock lock(&mu_); GPR_ASSERT(static_cast(uuid) <= entities_.size()); GPR_ASSERT(entities_[uuid - 1].type == type); entities_[uuid - 1].object = nullptr; @@ -69,7 +70,7 @@ void ChannelzRegistry::InternalUnregisterEntry(intptr_t uuid, EntityType type) { } void* ChannelzRegistry::InternalGetEntry(intptr_t uuid, EntityType type) { - mu_guard guard(&mu_); + MutexLock lock(&mu_); if (uuid < 1 || uuid > static_cast(entities_.size())) { return nullptr; } diff --git a/src/core/lib/gprpp/mutex_lock.h b/src/core/lib/gprpp/mutex_lock.h new file mode 100644 index 00000000000..54751d5fe46 --- /dev/null +++ b/src/core/lib/gprpp/mutex_lock.h @@ -0,0 +1,42 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_GPRPP_MUTEX_LOCK_H +#define GRPC_CORE_LIB_GPRPP_MUTEX_LOCK_H + +#include + +#include + +namespace grpc_core { + +class MutexLock { + public: + explicit MutexLock(gpr_mu* mu) : mu_(mu) { gpr_mu_lock(mu); } + ~MutexLock() { gpr_mu_unlock(mu_); } + + MutexLock(const MutexLock&) = delete; + MutexLock& operator=(const MutexLock&) = delete; + + private: + gpr_mu* const mu_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_GPRPP_MUTEX_LOCK_H */ diff --git a/src/core/lib/gprpp/orphanable.h b/src/core/lib/gprpp/orphanable.h index d0ec9b6461d..3123e3f5a39 100644 --- a/src/core/lib/gprpp/orphanable.h +++ b/src/core/lib/gprpp/orphanable.h @@ -86,7 +86,8 @@ class InternallyRefCounted : public Orphanable { GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE // Allow RefCountedPtr<> to access Unref() and IncrementRefCount(). - friend class RefCountedPtr; + template + friend class RefCountedPtr; InternallyRefCounted() { gpr_ref_init(&refs_, 1); } virtual ~InternallyRefCounted() {} @@ -129,7 +130,8 @@ class InternallyRefCountedWithTracing : public Orphanable { GPRC_ALLOW_CLASS_TO_USE_NON_PUBLIC_DELETE // Allow RefCountedPtr<> to access Unref() and IncrementRefCount(). - friend class RefCountedPtr; + template + friend class RefCountedPtr; InternallyRefCountedWithTracing() : InternallyRefCountedWithTracing(static_cast(nullptr)) {} diff --git a/src/core/lib/gprpp/ref_counted.h b/src/core/lib/gprpp/ref_counted.h index ddac5bd4755..03c293f6ed9 100644 --- a/src/core/lib/gprpp/ref_counted.h +++ b/src/core/lib/gprpp/ref_counted.h @@ -73,7 +73,8 @@ class RefCounted { private: // Allow RefCountedPtr<> to access IncrementRefCount(). - friend class RefCountedPtr; + template + friend class RefCountedPtr; void IncrementRefCount() { gpr_ref(&refs_); } @@ -152,7 +153,8 @@ class RefCountedWithTracing { private: // Allow RefCountedPtr<> to access IncrementRefCount(). - friend class RefCountedPtr; + template + friend class RefCountedPtr; void IncrementRefCount() { gpr_ref(&refs_); } diff --git a/src/core/lib/gprpp/ref_counted_ptr.h b/src/core/lib/gprpp/ref_counted_ptr.h index 534d3d03cb4..c2dfbdd90f0 100644 --- a/src/core/lib/gprpp/ref_counted_ptr.h +++ b/src/core/lib/gprpp/ref_counted_ptr.h @@ -36,25 +36,49 @@ class RefCountedPtr { RefCountedPtr(std::nullptr_t) {} // If value is non-null, we take ownership of a ref to it. - explicit RefCountedPtr(T* value) { value_ = value; } + template + explicit RefCountedPtr(Y* value) { + value_ = value; + } - // Move support. + // Move ctors. RefCountedPtr(RefCountedPtr&& other) { value_ = other.value_; other.value_ = nullptr; } + template + RefCountedPtr(RefCountedPtr&& other) { + value_ = other.value_; + other.value_ = nullptr; + } + + // Move assignment. RefCountedPtr& operator=(RefCountedPtr&& other) { if (value_ != nullptr) value_->Unref(); value_ = other.value_; other.value_ = nullptr; return *this; } + template + RefCountedPtr& operator=(RefCountedPtr&& other) { + if (value_ != nullptr) value_->Unref(); + value_ = other.value_; + other.value_ = nullptr; + return *this; + } - // Copy support. + // Copy ctors. RefCountedPtr(const RefCountedPtr& other) { if (other.value_ != nullptr) other.value_->IncrementRefCount(); value_ = other.value_; } + template + RefCountedPtr(const RefCountedPtr& other) { + if (other.value_ != nullptr) other.value_->IncrementRefCount(); + value_ = other.value_; + } + + // Copy assignment. RefCountedPtr& operator=(const RefCountedPtr& other) { // Note: Order of reffing and unreffing is important here in case value_ // and other.value_ are the same object. @@ -63,17 +87,32 @@ class RefCountedPtr { value_ = other.value_; return *this; } + template + RefCountedPtr& operator=(const RefCountedPtr& other) { + // Note: Order of reffing and unreffing is important here in case value_ + // and other.value_ are the same object. + if (other.value_ != nullptr) other.value_->IncrementRefCount(); + if (value_ != nullptr) value_->Unref(); + value_ = other.value_; + return *this; + } ~RefCountedPtr() { if (value_ != nullptr) value_->Unref(); } // If value is non-null, we take ownership of a ref to it. - void reset(T* value = nullptr) { + template + void reset(Y* value) { if (value_ != nullptr) value_->Unref(); value_ = value; } + void reset() { + if (value_ != nullptr) value_->Unref(); + value_ = nullptr; + } + // TODO(roth): This method exists solely as a transition mechanism to allow // us to pass a ref to idiomatic C code that does not use RefCountedPtr<>. // Once all of our code has been converted to idiomatic C++, this @@ -89,16 +128,34 @@ class RefCountedPtr { T& operator*() const { return *value_; } T* operator->() const { return value_; } - bool operator==(const RefCountedPtr& other) const { + template + bool operator==(const RefCountedPtr& other) const { return value_ == other.value_; } - bool operator==(const T* other) const { return value_ == other; } - bool operator!=(const RefCountedPtr& other) const { + + template + bool operator==(const Y* other) const { + return value_ == other; + } + + bool operator==(std::nullptr_t) const { return value_ == nullptr; } + + template + bool operator!=(const RefCountedPtr& other) const { return value_ != other.value_; } - bool operator!=(const T* other) const { return value_ != other; } + + template + bool operator!=(const Y* other) const { + return value_ != other; + } + + bool operator!=(std::nullptr_t) const { return value_ != nullptr; } private: + template + friend class RefCountedPtr; + T* value_ = nullptr; }; @@ -107,11 +164,6 @@ inline RefCountedPtr MakeRefCounted(Args&&... args) { return RefCountedPtr(New(std::forward(args)...)); } -template -inline RefCountedPtr MakePolymorphicRefCounted(Args&&... args) { - return RefCountedPtr(New(std::forward(args)...)); -} - } // namespace grpc_core #endif /* GRPC_CORE_LIB_GPRPP_REF_COUNTED_PTR_H */ diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h index 641fa180820..6f7ddd40435 100644 --- a/src/core/lib/iomgr/call_combiner.h +++ b/src/core/lib/iomgr/call_combiner.h @@ -102,7 +102,10 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner, /// If \a closure is NULL, then no closure will be invoked on /// cancellation; this effectively unregisters the previously set closure. /// However, most filters will not need to explicitly unregister their -/// callbacks, as this is done automatically when the call is destroyed. +/// callbacks, as this is done automatically when the call is destroyed. Filters +/// that schedule the cancellation closure on ExecCtx do not need to take a ref +/// on the call stack to guarantee closure liveness. This is done by explicitly +/// flushing ExecCtx after the unregistration during call destruction. void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner, grpc_closure* closure); diff --git a/src/core/lib/iomgr/ev_epoll1_linux.cc b/src/core/lib/iomgr/ev_epoll1_linux.cc index 86a0243d2ef..66e0f1fd6db 100644 --- a/src/core/lib/iomgr/ev_epoll1_linux.cc +++ b/src/core/lib/iomgr/ev_epoll1_linux.cc @@ -140,10 +140,6 @@ struct grpc_fd { struct grpc_fd* freelist_next; - /* The pollset that last noticed that the fd is readable. The actual type - * stored in this is (grpc_pollset *) */ - gpr_atm read_notifier_pollset; - grpc_iomgr_object iomgr_object; }; @@ -293,7 +289,6 @@ static grpc_fd* fd_create(int fd, const char* name, bool track_err) { new_fd->read_closure->InitEvent(); new_fd->write_closure->InitEvent(); new_fd->error_closure->InitEvent(); - gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = nullptr; @@ -376,11 +371,6 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd, gpr_mu_unlock(&fd_freelist_mu); } -static grpc_pollset* fd_get_read_notifier_pollset(grpc_fd* fd) { - gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); - return (grpc_pollset*)notifier; -} - static bool fd_is_shutdown(grpc_fd* fd) { return fd->read_closure->IsShutdown(); } @@ -397,11 +387,7 @@ static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) { fd->error_closure->NotifyOn(closure); } -static void fd_become_readable(grpc_fd* fd, grpc_pollset* notifier) { - fd->read_closure->SetReady(); - /* Use release store to match with acquire load in fd_get_read_notifier */ - gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); -} +static void fd_become_readable(grpc_fd* fd) { fd->read_closure->SetReady(); } static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); } @@ -642,7 +628,7 @@ static grpc_error* process_epoll_events(grpc_pollset* pollset) { } if (read_ev || cancel || err_fallback) { - fd_become_readable(fd, pollset); + fd_become_readable(fd); } if (write_ev || cancel || err_fallback) { @@ -1217,8 +1203,10 @@ static const grpc_event_engine_vtable vtable = { fd_notify_on_read, fd_notify_on_write, fd_notify_on_error, + fd_become_readable, + fd_become_writable, + fd_has_errors, fd_is_shutdown, - fd_get_read_notifier_pollset, pollset_init, pollset_shutdown, diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 7b368410cf6..b082634af11 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -46,6 +46,7 @@ #include "src/core/lib/gpr/tls.h" #include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/manual_constructor.h" +#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/lib/iomgr/block_annotate.h" #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/is_epollexclusive_available.h" @@ -135,7 +136,7 @@ struct pollable { // underlying epoll set (i.e whenever fd_orphan() is called). // // Implementing (2) above (i.e removing fds from cache on fd_orphan) adds a - // lot of complexity since an fd can be present in multiple pollalbles. So our + // lot of complexity since an fd can be present in multiple pollables. So our // implementation ONLY DOES (1) and NOT (2). // // The cache_fd.salt variable helps here to maintain correctness (it serves as @@ -220,10 +221,6 @@ struct grpc_fd { struct grpc_fd* freelist_next; grpc_closure* on_done_closure; - // The pollset that last noticed that the fd is readable. The actual type - // stored in this is (grpc_pollset *) - gpr_atm read_notifier_pollset; - grpc_iomgr_object iomgr_object; // Do we need to track EPOLLERR events separately? @@ -353,7 +350,6 @@ static void invalidate_fd(grpc_fd* fd) { memset(&fd->pollable_mu, -1, sizeof(fd->pollable_mu)); fd->pollable_obj = nullptr; fd->on_done_closure = nullptr; - gpr_atm_no_barrier_store(&fd->read_notifier_pollset, 0); memset(&fd->iomgr_object, -1, sizeof(fd->iomgr_object)); fd->track_err = false; } @@ -445,7 +441,6 @@ static grpc_fd* fd_create(int fd, const char* name, bool track_err) { new_fd->error_closure->InitEvent(); new_fd->freelist_next = nullptr; new_fd->on_done_closure = nullptr; - gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); char* fd_name; gpr_asprintf(&fd_name, "%s fd=%d", name, fd); @@ -514,11 +509,6 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd, UNREF_BY(fd, 2, reason); /* Drop the reference */ } -static grpc_pollset* fd_get_read_notifier_pollset(grpc_fd* fd) { - gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); - return (grpc_pollset*)notifier; -} - static bool fd_is_shutdown(grpc_fd* fd) { return fd->read_closure->IsShutdown(); } @@ -746,7 +736,7 @@ static void pollset_maybe_finish_shutdown(grpc_pollset* pollset) { static grpc_error* kick_one_worker(grpc_pollset_worker* specific_worker) { GPR_TIMER_SCOPE("kick_one_worker", 0); pollable* p = specific_worker->pollable_obj; - grpc_core::mu_guard lock(&p->mu); + grpc_core::MutexLock lock(&p->mu); GPR_ASSERT(specific_worker != nullptr); if (specific_worker->kicked) { if (grpc_polling_trace.enabled()) { @@ -875,17 +865,7 @@ static int poll_deadline_to_millis_timeout(grpc_millis millis) { return static_cast(delta); } -static void fd_become_readable(grpc_fd* fd, grpc_pollset* notifier) { - fd->read_closure->SetReady(); - - /* Note, it is possible that fd_become_readable might be called twice with - different 'notifier's when an fd becomes readable and it is in two epoll - sets (This can happen briefly during polling island merges). In such cases - it does not really matter which notifer is set as the read_notifier_pollset - (They would both point to the same polling island anyway) */ - /* Use release store to match with acquire load in fd_get_read_notifier */ - gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); -} +static void fd_become_readable(grpc_fd* fd) { fd->read_closure->SetReady(); } static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); } @@ -983,7 +963,7 @@ static grpc_error* pollable_process_events(grpc_pollset* pollset, fd_has_errors(fd); } if (read_ev || cancel || err_fallback) { - fd_become_readable(fd, pollset); + fd_become_readable(fd); } if (write_ev || cancel || err_fallback) { fd_become_writable(fd); @@ -1636,8 +1616,10 @@ static const grpc_event_engine_vtable vtable = { fd_notify_on_read, fd_notify_on_write, fd_notify_on_error, + fd_become_readable, + fd_become_writable, + fd_has_errors, fd_is_shutdown, - fd_get_read_notifier_pollset, pollset_init, pollset_shutdown, diff --git a/src/core/lib/iomgr/ev_epollsig_linux.cc b/src/core/lib/iomgr/ev_epollsig_linux.cc index 2189801c187..5695ac795d7 100644 --- a/src/core/lib/iomgr/ev_epollsig_linux.cc +++ b/src/core/lib/iomgr/ev_epollsig_linux.cc @@ -137,10 +137,6 @@ struct grpc_fd { struct grpc_fd* freelist_next; grpc_closure* on_done_closure; - /* The pollset that last noticed that the fd is readable. The actual type - * stored in this is (grpc_pollset *) */ - gpr_atm read_notifier_pollset; - grpc_iomgr_object iomgr_object; /* Do we need to track EPOLLERR events separately? */ @@ -845,7 +841,6 @@ static grpc_fd* fd_create(int fd, const char* name, bool track_err) { new_fd->write_closure->InitEvent(); new_fd->error_closure->InitEvent(); new_fd->track_err = track_err; - gpr_atm_no_barrier_store(&new_fd->read_notifier_pollset, (gpr_atm)NULL); new_fd->freelist_next = nullptr; new_fd->on_done_closure = nullptr; @@ -927,11 +922,6 @@ static void fd_orphan(grpc_fd* fd, grpc_closure* on_done, int* release_fd, GRPC_ERROR_UNREF(error); } -static grpc_pollset* fd_get_read_notifier_pollset(grpc_fd* fd) { - gpr_atm notifier = gpr_atm_acq_load(&fd->read_notifier_pollset); - return (grpc_pollset*)notifier; -} - static bool fd_is_shutdown(grpc_fd* fd) { return fd->read_closure->IsShutdown(); } @@ -958,6 +948,12 @@ static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) { fd->error_closure->NotifyOn(closure); } +static void fd_become_readable(grpc_fd* fd) { fd->read_closure->SetReady(); } + +static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); } + +static void fd_has_errors(grpc_fd* fd) { fd->error_closure->SetReady(); } + /******************************************************************************* * Pollset Definitions */ @@ -1115,22 +1111,6 @@ static int poll_deadline_to_millis_timeout(grpc_millis millis) { return static_cast(delta); } -static void fd_become_readable(grpc_fd* fd, grpc_pollset* notifier) { - fd->read_closure->SetReady(); - - /* Note, it is possible that fd_become_readable might be called twice with - different 'notifier's when an fd becomes readable and it is in two epoll - sets (This can happen briefly during polling island merges). In such cases - it does not really matter which notifer is set as the read_notifier_pollset - (They would both point to the same polling island anyway) */ - /* Use release store to match with acquire load in fd_get_read_notifier */ - gpr_atm_rel_store(&fd->read_notifier_pollset, (gpr_atm)notifier); -} - -static void fd_become_writable(grpc_fd* fd) { fd->write_closure->SetReady(); } - -static void fd_has_errors(grpc_fd* fd) { fd->error_closure->SetReady(); } - static void pollset_release_polling_island(grpc_pollset* ps, const char* reason) { if (ps->po.pi != nullptr) { @@ -1283,7 +1263,7 @@ static void pollset_work_and_unlock(grpc_pollset* pollset, fd_has_errors(fd); } if (read_ev || cancel || err_fallback) { - fd_become_readable(fd, pollset); + fd_become_readable(fd); } if (write_ev || cancel || err_fallback) { fd_become_writable(fd); @@ -1667,8 +1647,10 @@ static const grpc_event_engine_vtable vtable = { fd_notify_on_read, fd_notify_on_write, fd_notify_on_error, + fd_become_readable, + fd_become_writable, + fd_has_errors, fd_is_shutdown, - fd_get_read_notifier_pollset, pollset_init, pollset_shutdown, diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc index c9c09881a2a..fb4c71ef71b 100644 --- a/src/core/lib/iomgr/ev_poll_posix.cc +++ b/src/core/lib/iomgr/ev_poll_posix.cc @@ -108,9 +108,6 @@ struct grpc_fd { grpc_closure* on_done_closure; grpc_iomgr_object iomgr_object; - - /* The pollset that last noticed and notified that the fd is readable */ - grpc_pollset* read_notifier_pollset; }; /* Begin polling on an fd. @@ -131,8 +128,7 @@ static uint32_t fd_begin_poll(grpc_fd* fd, grpc_pollset* pollset, MUST NOT be called with a pollset lock taken if got_read or got_write are 1, also does the become_{readable,writable} as appropriate. */ -static void fd_end_poll(grpc_fd_watcher* rec, int got_read, int got_write, - grpc_pollset* read_notifier_pollset); +static void fd_end_poll(grpc_fd_watcher* rec, int got_read, int got_write); /* Return 1 if this fd is orphaned, 0 otherwise */ static bool fd_is_orphaned(grpc_fd* fd); @@ -346,7 +342,6 @@ static grpc_fd* fd_create(int fd, const char* name, bool track_err) { r->closed = 0; r->released = 0; gpr_atm_no_barrier_store(&r->pollhup, 0); - r->read_notifier_pollset = nullptr; char* name2; gpr_asprintf(&name2, "%s fd=%d", name, fd); @@ -359,17 +354,6 @@ static bool fd_is_orphaned(grpc_fd* fd) { return (gpr_atm_acq_load(&fd->refst) & 1) == 0; } -/* Return the read-notifier pollset */ -static grpc_pollset* fd_get_read_notifier_pollset(grpc_fd* fd) { - grpc_pollset* notifier = nullptr; - - gpr_mu_lock(&fd->mu); - notifier = fd->read_notifier_pollset; - gpr_mu_unlock(&fd->mu); - - return notifier; -} - static grpc_error* pollset_kick_locked(grpc_fd_watcher* watcher) { gpr_mu_lock(&watcher->pollset->mu); GPR_ASSERT(watcher->worker); @@ -512,11 +496,6 @@ static int set_ready_locked(grpc_fd* fd, grpc_closure** st) { } } -static void set_read_notifier_pollset_locked( - grpc_fd* fd, grpc_pollset* read_notifier_pollset) { - fd->read_notifier_pollset = read_notifier_pollset; -} - static void fd_shutdown(grpc_fd* fd, grpc_error* why) { gpr_mu_lock(&fd->mu); /* only shutdown once */ @@ -553,8 +532,28 @@ static void fd_notify_on_write(grpc_fd* fd, grpc_closure* closure) { } static void fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) { - gpr_log(GPR_ERROR, "Polling engine does not support tracking errors."); - abort(); + if (grpc_polling_trace.enabled()) { + gpr_log(GPR_ERROR, "Polling engine does not support tracking errors."); + } + GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_CANCELLED); +} + +static void fd_set_readable(grpc_fd* fd) { + gpr_mu_lock(&fd->mu); + set_ready_locked(fd, &fd->read_closure); + gpr_mu_unlock(&fd->mu); +} + +static void fd_set_writable(grpc_fd* fd) { + gpr_mu_lock(&fd->mu); + set_ready_locked(fd, &fd->write_closure); + gpr_mu_unlock(&fd->mu); +} + +static void fd_set_error(grpc_fd* fd) { + if (grpc_polling_trace.enabled()) { + gpr_log(GPR_ERROR, "Polling engine does not support tracking errors."); + } } static uint32_t fd_begin_poll(grpc_fd* fd, grpc_pollset* pollset, @@ -608,8 +607,7 @@ static uint32_t fd_begin_poll(grpc_fd* fd, grpc_pollset* pollset, return mask; } -static void fd_end_poll(grpc_fd_watcher* watcher, int got_read, int got_write, - grpc_pollset* read_notifier_pollset) { +static void fd_end_poll(grpc_fd_watcher* watcher, int got_read, int got_write) { int was_polling = 0; int kick = 0; grpc_fd* fd = watcher->fd; @@ -645,9 +643,6 @@ static void fd_end_poll(grpc_fd_watcher* watcher, int got_read, int got_write, if (set_ready_locked(fd, &fd->read_closure)) { kick = 1; } - if (read_notifier_pollset != nullptr) { - set_read_notifier_pollset_locked(fd, read_notifier_pollset); - } } if (got_write) { if (set_ready_locked(fd, &fd->write_closure)) { @@ -997,16 +992,16 @@ static grpc_error* pollset_work(grpc_pollset* pollset, for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == nullptr) { - fd_end_poll(&watchers[i], 0, 0, nullptr); + fd_end_poll(&watchers[i], 0, 0); } else { // Wake up all the file descriptors, if we have an invalid one // we can identify it on the next pollset_work() - fd_end_poll(&watchers[i], 1, 1, pollset); + fd_end_poll(&watchers[i], 1, 1); } } } else if (r == 0) { for (i = 1; i < pfd_count; i++) { - fd_end_poll(&watchers[i], 0, 0, nullptr); + fd_end_poll(&watchers[i], 0, 0); } } else { if (pfds[0].revents & POLLIN_CHECK) { @@ -1018,7 +1013,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, } for (i = 1; i < pfd_count; i++) { if (watchers[i].fd == nullptr) { - fd_end_poll(&watchers[i], 0, 0, nullptr); + fd_end_poll(&watchers[i], 0, 0); } else { if (grpc_polling_trace.enabled()) { gpr_log(GPR_INFO, "%p got_event: %d r:%d w:%d [%d]", pollset, @@ -1032,7 +1027,7 @@ static grpc_error* pollset_work(grpc_pollset* pollset, gpr_atm_no_barrier_store(&watchers[i].fd->pollhup, 1); } fd_end_poll(&watchers[i], pfds[i].revents & POLLIN_CHECK, - pfds[i].revents & POLLOUT_CHECK, pollset); + pfds[i].revents & POLLOUT_CHECK); } } } @@ -1723,8 +1718,10 @@ static const grpc_event_engine_vtable vtable = { fd_notify_on_read, fd_notify_on_write, fd_notify_on_error, + fd_set_readable, + fd_set_writable, + fd_set_error, fd_is_shutdown, - fd_get_read_notifier_pollset, pollset_init, pollset_shutdown, diff --git a/src/core/lib/iomgr/ev_posix.cc b/src/core/lib/iomgr/ev_posix.cc index 1139b3273ab..0205363d5c4 100644 --- a/src/core/lib/iomgr/ev_posix.cc +++ b/src/core/lib/iomgr/ev_posix.cc @@ -59,7 +59,14 @@ grpc_core::DebugOnlyTraceFlag grpc_polling_api_trace(false, "polling_api"); /** Default poll() function - a pointer so that it can be overridden by some * tests */ +#ifndef GPR_AIX grpc_poll_function_type grpc_poll_function = poll; +#else +int aix_poll(struct pollfd fds[], nfds_t nfds, int timeout) { + return poll(fds, nfds, timeout); +} +grpc_poll_function_type grpc_poll_function = aix_poll; +#endif grpc_wakeup_fd grpc_global_wakeup_fd; @@ -101,10 +108,28 @@ const grpc_event_engine_vtable* init_non_polling(bool explicit_request) { } } // namespace -static const event_engine_factory g_factories[] = { +#define ENGINE_HEAD_CUSTOM "head_custom" +#define ENGINE_TAIL_CUSTOM "tail_custom" + +// The global array of event-engine factories. Each entry is a pair with a name +// and an event-engine generator function (nullptr if there is no generator +// registered for this name). The middle entries are the engines predefined by +// open-source gRPC. The head entries represent an opportunity for specific +// high-priority custom pollers to be added by the initializer plugins of +// custom-built gRPC libraries. The tail entries represent the same, but for +// low-priority custom pollers. The actual poller selected is either the first +// available one in the list if no specific poller is requested, or the first +// specific poller that is requested by name in the GRPC_POLL_STRATEGY +// environment variable if that variable is set (which should be a +// comma-separated list of one or more event engine names) +static event_engine_factory g_factories[] = { + {ENGINE_HEAD_CUSTOM, nullptr}, {ENGINE_HEAD_CUSTOM, nullptr}, + {ENGINE_HEAD_CUSTOM, nullptr}, {ENGINE_HEAD_CUSTOM, nullptr}, {"epollex", grpc_init_epollex_linux}, {"epoll1", grpc_init_epoll1_linux}, {"epollsig", grpc_init_epollsig_linux}, {"poll", grpc_init_poll_posix}, {"poll-cv", grpc_init_poll_cv_posix}, {"none", init_non_polling}, + {ENGINE_TAIL_CUSTOM, nullptr}, {ENGINE_TAIL_CUSTOM, nullptr}, + {ENGINE_TAIL_CUSTOM, nullptr}, {ENGINE_TAIL_CUSTOM, nullptr}, }; static void add(const char* beg, const char* end, char*** ss, size_t* ns) { @@ -138,7 +163,7 @@ static bool is(const char* want, const char* have) { static void try_engine(const char* engine) { for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { - if (is(engine, g_factories[i].name)) { + if (g_factories[i].factory != nullptr && is(engine, g_factories[i].name)) { if ((g_event_engine = g_factories[i].factory( 0 == strcmp(engine, g_factories[i].name)))) { g_poll_strategy_name = g_factories[i].name; @@ -149,14 +174,32 @@ static void try_engine(const char* engine) { } } -/* This should be used for testing purposes ONLY */ -void grpc_set_event_engine_test_only( - const grpc_event_engine_vtable* ev_engine) { - g_event_engine = ev_engine; -} +/* Call this before calling grpc_event_engine_init() */ +void grpc_register_event_engine_factory(const char* name, + event_engine_factory_fn factory, + bool add_at_head) { + const char* custom_match = + add_at_head ? ENGINE_HEAD_CUSTOM : ENGINE_TAIL_CUSTOM; + + // Overwrite an existing registration if already registered + for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { + if (0 == strcmp(name, g_factories[i].name)) { + g_factories[i].factory = factory; + return; + } + } -const grpc_event_engine_vtable* grpc_get_event_engine_test_only() { - return g_event_engine; + // Otherwise fill in an available custom slot + for (size_t i = 0; i < GPR_ARRAY_SIZE(g_factories); i++) { + if (0 == strcmp(g_factories[i].name, custom_match)) { + g_factories[i].name = name; + g_factories[i].factory = factory; + return; + } + } + + // Otherwise fail + GPR_ASSERT(false); } /* Call this only after calling grpc_event_engine_init() */ @@ -239,6 +282,12 @@ void grpc_fd_notify_on_error(grpc_fd* fd, grpc_closure* closure) { g_event_engine->fd_notify_on_error(fd, closure); } +void grpc_fd_set_readable(grpc_fd* fd) { g_event_engine->fd_set_readable(fd); } + +void grpc_fd_set_writable(grpc_fd* fd) { g_event_engine->fd_set_writable(fd); } + +void grpc_fd_set_error(grpc_fd* fd) { g_event_engine->fd_set_error(fd); } + static size_t pollset_size(void) { return g_event_engine->pollset_size; } static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) { diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h index b4c17fc80df..b8fb8f534b4 100644 --- a/src/core/lib/iomgr/ev_posix.h +++ b/src/core/lib/iomgr/ev_posix.h @@ -51,8 +51,10 @@ typedef struct grpc_event_engine_vtable { void (*fd_notify_on_read)(grpc_fd* fd, grpc_closure* closure); void (*fd_notify_on_write)(grpc_fd* fd, grpc_closure* closure); void (*fd_notify_on_error)(grpc_fd* fd, grpc_closure* closure); + void (*fd_set_readable)(grpc_fd* fd); + void (*fd_set_writable)(grpc_fd* fd); + void (*fd_set_error)(grpc_fd* fd); bool (*fd_is_shutdown)(grpc_fd* fd); - grpc_pollset* (*fd_get_read_notifier_pollset)(grpc_fd* fd); void (*pollset_init)(grpc_pollset* pollset, gpr_mu** mu); void (*pollset_shutdown)(grpc_pollset* pollset, grpc_closure* closure); @@ -80,6 +82,11 @@ typedef struct grpc_event_engine_vtable { void (*shutdown_engine)(void); } grpc_event_engine_vtable; +/* register a new event engine factory */ +void grpc_register_event_engine_factory( + const char* name, const grpc_event_engine_vtable* (*factory)(bool), + bool add_at_head); + void grpc_event_engine_init(void); void grpc_event_engine_shutdown(void); @@ -142,8 +149,20 @@ void grpc_fd_notify_on_write(grpc_fd* fd, grpc_closure* closure); * needs to have been set on grpc_fd_create */ void grpc_fd_notify_on_error(grpc_fd* fd, grpc_closure* closure); -/* Return the read notifier pollset from the fd */ -grpc_pollset* grpc_fd_get_read_notifier_pollset(grpc_fd* fd); +/* Forcibly set the fd to be readable, resulting in the closure registered with + * grpc_fd_notify_on_read being invoked. + */ +void grpc_fd_set_readable(grpc_fd* fd); + +/* Forcibly set the fd to be writable, resulting in the closure registered with + * grpc_fd_notify_on_write being invoked. + */ +void grpc_fd_set_writable(grpc_fd* fd); + +/* Forcibly set the fd to have errored, resulting in the closure registered with + * grpc_fd_notify_on_error being invoked. + */ +void grpc_fd_set_error(grpc_fd* fd); /* pollset_posix functions */ @@ -159,9 +178,4 @@ void grpc_pollset_set_del_fd(grpc_pollset_set* pollset_set, grpc_fd* fd); typedef int (*grpc_poll_function_type)(struct pollfd*, nfds_t, int); extern grpc_poll_function_type grpc_poll_function; -/* WARNING: The following two functions should be used for testing purposes - * ONLY */ -void grpc_set_event_engine_test_only(const grpc_event_engine_vtable*); -const grpc_event_engine_vtable* grpc_get_event_engine_test_only(); - #endif /* GRPC_CORE_LIB_IOMGR_EV_POSIX_H */ diff --git a/src/core/lib/iomgr/iocp_windows.cc b/src/core/lib/iomgr/iocp_windows.cc index ce772310362..ad325fe2156 100644 --- a/src/core/lib/iomgr/iocp_windows.cc +++ b/src/core/lib/iomgr/iocp_windows.cc @@ -89,10 +89,15 @@ grpc_iocp_work_status grpc_iocp_work(grpc_millis deadline) { } else { abort(); } - success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, - FALSE, &flags); - info->bytes_transfered = bytes; - info->wsa_error = success ? 0 : WSAGetLastError(); + if (socket->shutdown_called) { + info->bytes_transfered = 0; + info->wsa_error = WSA_OPERATION_ABORTED; + } else { + success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, + FALSE, &flags); + info->bytes_transfered = bytes; + info->wsa_error = success ? 0 : WSAGetLastError(); + } GPR_ASSERT(overlapped == &info->overlapped); grpc_socket_become_ready(socket, info); return GRPC_IOCP_WORK_WORK; diff --git a/src/core/lib/iomgr/iomgr_posix_cfstream.cc b/src/core/lib/iomgr/iomgr_posix_cfstream.cc new file mode 100644 index 00000000000..235a9e0712f --- /dev/null +++ b/src/core/lib/iomgr/iomgr_posix_cfstream.cc @@ -0,0 +1,75 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include "src/core/lib/iomgr/port.h" + +#ifdef GRPC_CFSTREAM_IOMGR + +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/iomgr/ev_posix.h" +#include "src/core/lib/iomgr/iomgr_internal.h" +#include "src/core/lib/iomgr/iomgr_posix.h" +#include "src/core/lib/iomgr/resolve_address.h" +#include "src/core/lib/iomgr/tcp_client.h" +#include "src/core/lib/iomgr/tcp_posix.h" +#include "src/core/lib/iomgr/tcp_server.h" +#include "src/core/lib/iomgr/timer.h" + +static const char* grpc_cfstream_env_var = "grpc_cfstream"; + +extern grpc_tcp_server_vtable grpc_posix_tcp_server_vtable; +extern grpc_tcp_client_vtable grpc_posix_tcp_client_vtable; +extern grpc_tcp_client_vtable grpc_cfstream_client_vtable; +extern grpc_timer_vtable grpc_generic_timer_vtable; +extern grpc_pollset_vtable grpc_posix_pollset_vtable; +extern grpc_pollset_set_vtable grpc_posix_pollset_set_vtable; +extern grpc_address_resolver_vtable grpc_posix_resolver_vtable; + +static void iomgr_platform_init(void) { + grpc_wakeup_fd_global_init(); + grpc_event_engine_init(); +} + +static void iomgr_platform_flush(void) {} + +static void iomgr_platform_shutdown(void) { + grpc_event_engine_shutdown(); + grpc_wakeup_fd_global_destroy(); +} + +static grpc_iomgr_platform_vtable vtable = { + iomgr_platform_init, iomgr_platform_flush, iomgr_platform_shutdown}; + +void grpc_set_default_iomgr_platform() { + char* enable_cfstream = getenv(grpc_cfstream_env_var); + grpc_tcp_client_vtable* client_vtable = &grpc_posix_tcp_client_vtable; + if (enable_cfstream != nullptr && enable_cfstream[0] == '1') { + client_vtable = &grpc_cfstream_client_vtable; + } + grpc_set_tcp_client_impl(client_vtable); + grpc_set_tcp_server_impl(&grpc_posix_tcp_server_vtable); + grpc_set_timer_impl(&grpc_generic_timer_vtable); + grpc_set_pollset_vtable(&grpc_posix_pollset_vtable); + grpc_set_pollset_set_vtable(&grpc_posix_pollset_set_vtable); + grpc_set_resolver_impl(&grpc_posix_resolver_vtable); + grpc_set_iomgr_platform_vtable(&vtable); +} + +#endif /* GRPC_CFSTREAM_IOMGR */ diff --git a/src/core/lib/iomgr/port.h b/src/core/lib/iomgr/port.h index 80d8e63cdd0..066417b93cc 100644 --- a/src/core/lib/iomgr/port.h +++ b/src/core/lib/iomgr/port.h @@ -98,9 +98,9 @@ #define GRPC_POSIX_FORK 1 #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1 #ifdef GRPC_CFSTREAM -#define GRPC_POSIX_SOCKET_IOMGR 1 -#define GRPC_CFSTREAM_ENDPOINT 1 +#define GRPC_CFSTREAM_IOMGR 1 #define GRPC_CFSTREAM_CLIENT 1 +#define GRPC_CFSTREAM_ENDPOINT 1 #define GRPC_POSIX_SOCKET_ARES_EV_DRIVER 1 #define GRPC_POSIX_SOCKET_EV 1 #define GRPC_POSIX_SOCKET_EV_EPOLL1 1 @@ -111,6 +111,7 @@ #define GRPC_POSIX_SOCKET_SOCKADDR 1 #define GRPC_POSIX_SOCKET_SOCKET_FACTORY 1 #define GRPC_POSIX_SOCKET_TCP 1 +#define GRPC_POSIX_SOCKET_TCP_CLIENT 1 #define GRPC_POSIX_SOCKET_TCP_SERVER 1 #define GRPC_POSIX_SOCKET_TCP_SERVER_UTILS_COMMON 1 #define GRPC_POSIX_SOCKET_UTILS_COMMON 1 @@ -139,6 +140,18 @@ #define GRPC_POSIX_SOCKET 1 #define GRPC_POSIX_SOCKETUTILS 1 #define GRPC_POSIX_WAKEUP_FD 1 +#elif defined(GPR_SOLARIS) +#define GRPC_HAVE_UNIX_SOCKET 1 +#define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1 +#define GRPC_POSIX_SOCKET 1 +#define GRPC_POSIX_SOCKETUTILS 1 +#define GRPC_POSIX_WAKEUP_FD 1 +#elif defined(GPR_AIX) +#define GRPC_HAVE_UNIX_SOCKET 1 +#define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1 +#define GRPC_POSIX_SOCKET 1 +#define GRPC_POSIX_SOCKETUTILS 1 +#define GRPC_POSIX_WAKEUP_FD 1 #elif defined(GPR_NACL) #define GRPC_HAVE_ARPA_NAMESER 1 #define GRPC_POSIX_NO_SPECIAL_WAKEUP_FD 1 diff --git a/src/core/lib/iomgr/resource_quota.cc b/src/core/lib/iomgr/resource_quota.cc index 539bc120cec..b6fc7579f7e 100644 --- a/src/core/lib/iomgr/resource_quota.cc +++ b/src/core/lib/iomgr/resource_quota.cc @@ -96,6 +96,9 @@ struct grpc_resource_user { list, false otherwise */ bool added_to_free_pool; + /* The number of threads currently allocated to this resource user */ + gpr_atm num_threads_allocated; + /* Reclaimers: index 0 is the benign reclaimer, 1 is the destructive reclaimer */ grpc_closure* reclaimers[2]; @@ -135,12 +138,33 @@ struct grpc_resource_quota { gpr_atm last_size; + /* Mutex to protect max_threads and num_threads_allocated */ + /* Note: We could have used gpr_atm for max_threads and num_threads_allocated + * and avoid having this mutex; but in that case, each invocation of the + * function grpc_resource_user_allocate_threads() would have had to do at + * least two atomic loads (for max_threads and num_threads_allocated) followed + * by a CAS (on num_threads_allocated). + * Moreover, we expect grpc_resource_user_allocate_threads() to be often + * called concurrently thereby increasing the chances of failing the CAS + * operation. This additional complexity is not worth the tiny perf gain we + * may (or may not) have by using atomics */ + gpr_mu thread_count_mu; + + /* Max number of threads allowed */ + int max_threads; + + /* Number of threads currently allocated via this resource_quota object */ + int num_threads_allocated; + /* Has rq_step been scheduled to occur? */ bool step_scheduled; + /* Are we currently reclaiming memory */ bool reclaiming; + /* Closure around rq_step */ grpc_closure rq_step_closure; + /* Closure around rq_reclamation_done */ grpc_closure rq_reclamation_done_closure; @@ -524,6 +548,11 @@ static void ru_shutdown(void* ru, grpc_error* error) { static void ru_destroy(void* ru, grpc_error* error) { grpc_resource_user* resource_user = static_cast(ru); GPR_ASSERT(gpr_atm_no_barrier_load(&resource_user->refs) == 0); + // Free all the remaining thread quota + grpc_resource_user_free_threads(resource_user, + static_cast(gpr_atm_no_barrier_load( + &resource_user->num_threads_allocated))); + for (int i = 0; i < GRPC_RULIST_COUNT; i++) { rulist_remove(resource_user, static_cast(i)); } @@ -594,6 +623,9 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { resource_quota->free_pool = INT64_MAX; resource_quota->size = INT64_MAX; gpr_atm_no_barrier_store(&resource_quota->last_size, GPR_ATM_MAX); + gpr_mu_init(&resource_quota->thread_count_mu); + resource_quota->max_threads = INT_MAX; + resource_quota->num_threads_allocated = 0; resource_quota->step_scheduled = false; resource_quota->reclaiming = false; gpr_atm_no_barrier_store(&resource_quota->memory_usage_estimation, 0); @@ -616,6 +648,8 @@ grpc_resource_quota* grpc_resource_quota_create(const char* name) { void grpc_resource_quota_unref_internal(grpc_resource_quota* resource_quota) { if (gpr_unref(&resource_quota->refs)) { + // No outstanding thread quota + GPR_ASSERT(resource_quota->num_threads_allocated == 0); GRPC_COMBINER_UNREF(resource_quota->combiner, "resource_quota"); gpr_free(resource_quota->name); gpr_free(resource_quota); @@ -646,6 +680,15 @@ double grpc_resource_quota_get_memory_pressure( (static_cast(MEMORY_USAGE_ESTIMATION_MAX)); } +/* Public API */ +void grpc_resource_quota_set_max_threads(grpc_resource_quota* resource_quota, + int new_max_threads) { + GPR_ASSERT(new_max_threads >= 0); + gpr_mu_lock(&resource_quota->thread_count_mu); + resource_quota->max_threads = new_max_threads; + gpr_mu_unlock(&resource_quota->thread_count_mu); +} + /* Public API */ void grpc_resource_quota_resize(grpc_resource_quota* resource_quota, size_t size) { @@ -731,6 +774,7 @@ grpc_resource_user* grpc_resource_user_create( grpc_closure_list_init(&resource_user->on_allocated); resource_user->allocating = false; resource_user->added_to_free_pool = false; + gpr_atm_no_barrier_store(&resource_user->num_threads_allocated, 0); resource_user->reclaimers[0] = nullptr; resource_user->reclaimers[1] = nullptr; resource_user->new_reclaimers[0] = nullptr; @@ -785,6 +829,40 @@ void grpc_resource_user_shutdown(grpc_resource_user* resource_user) { } } +bool grpc_resource_user_allocate_threads(grpc_resource_user* resource_user, + int thread_count) { + GPR_ASSERT(thread_count >= 0); + bool is_success = false; + gpr_mu_lock(&resource_user->resource_quota->thread_count_mu); + grpc_resource_quota* rq = resource_user->resource_quota; + if (rq->num_threads_allocated + thread_count <= rq->max_threads) { + rq->num_threads_allocated += thread_count; + gpr_atm_no_barrier_fetch_add(&resource_user->num_threads_allocated, + thread_count); + is_success = true; + } + gpr_mu_unlock(&resource_user->resource_quota->thread_count_mu); + return is_success; +} + +void grpc_resource_user_free_threads(grpc_resource_user* resource_user, + int thread_count) { + GPR_ASSERT(thread_count >= 0); + gpr_mu_lock(&resource_user->resource_quota->thread_count_mu); + grpc_resource_quota* rq = resource_user->resource_quota; + rq->num_threads_allocated -= thread_count; + int old_count = static_cast(gpr_atm_no_barrier_fetch_add( + &resource_user->num_threads_allocated, -thread_count)); + if (old_count < thread_count || rq->num_threads_allocated < 0) { + gpr_log(GPR_ERROR, + "Releasing more threads (%d) than currently allocated (rq threads: " + "%d, ru threads: %d)", + thread_count, rq->num_threads_allocated + thread_count, old_count); + abort(); + } + gpr_mu_unlock(&resource_user->resource_quota->thread_count_mu); +} + void grpc_resource_user_alloc(grpc_resource_user* resource_user, size_t size, grpc_closure* optional_on_done) { gpr_mu_lock(&resource_user->mu); diff --git a/src/core/lib/iomgr/resource_quota.h b/src/core/lib/iomgr/resource_quota.h index 937daf87282..7b0ed7417a9 100644 --- a/src/core/lib/iomgr/resource_quota.h +++ b/src/core/lib/iomgr/resource_quota.h @@ -93,6 +93,22 @@ void grpc_resource_user_ref(grpc_resource_user* resource_user); void grpc_resource_user_unref(grpc_resource_user* resource_user); void grpc_resource_user_shutdown(grpc_resource_user* resource_user); +/* Attempts to get quota from the resource_user to create 'thread_count' number + * of threads. Returns true if successful (i.e the caller is now free to create + * 'thread_count' number of threads) or false if quota is not available */ +bool grpc_resource_user_allocate_threads(grpc_resource_user* resource_user, + int thread_count); +/* Releases 'thread_count' worth of quota back to the resource user. The quota + * should have been previously obtained successfully by calling + * grpc_resource_user_allocate_threads(). + * + * Note: There need not be an exact one-to-one correspondence between + * grpc_resource_user_allocate_threads() and grpc_resource_user_free_threads() + * calls. The only requirement is that the number of threads allocated should + * all be eventually released */ +void grpc_resource_user_free_threads(grpc_resource_user* resource_user, + int thread_count); + /* Allocate from the resource user (and its quota). If optional_on_done is NULL, then allocate immediately. This may push the quota over-limit, at which point reclamation will kick in. diff --git a/src/core/lib/iomgr/socket_windows.cc b/src/core/lib/iomgr/socket_windows.cc index 2e23409582b..999c6646ad4 100644 --- a/src/core/lib/iomgr/socket_windows.cc +++ b/src/core/lib/iomgr/socket_windows.cc @@ -36,6 +36,7 @@ #include "src/core/lib/iomgr/iomgr_internal.h" #include "src/core/lib/iomgr/pollset.h" #include "src/core/lib/iomgr/pollset_windows.h" +#include "src/core/lib/iomgr/sockaddr_windows.h" #include "src/core/lib/iomgr/socket_windows.h" grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name) { @@ -51,6 +52,10 @@ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name) { return r; } +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket) { + return socket->socket; +} + /* Schedule a shutdown of the socket operations. Will call the pending operations to abort them. We need to do that this way because of the various callsites of that function, which happens to be in various @@ -148,4 +153,32 @@ void grpc_socket_become_ready(grpc_winsocket* socket, if (should_destroy) destroy(socket); } +static gpr_once g_probe_ipv6_once = GPR_ONCE_INIT; +static bool g_ipv6_loopback_available = false; + +static void probe_ipv6_once(void) { + SOCKET s = socket(AF_INET6, SOCK_STREAM, 0); + g_ipv6_loopback_available = 0; + if (s == INVALID_SOCKET) { + gpr_log(GPR_INFO, "Disabling AF_INET6 sockets because socket() failed."); + } else { + grpc_sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_addr.s6_addr[15] = 1; /* [::1]:0 */ + if (bind(s, reinterpret_cast(&addr), sizeof(addr)) == 0) { + g_ipv6_loopback_available = 1; + } else { + gpr_log(GPR_INFO, + "Disabling AF_INET6 sockets because ::1 is not available."); + } + closesocket(s); + } +} + +int grpc_ipv6_loopback_available(void) { + gpr_once_init(&g_probe_ipv6_once, probe_ipv6_once); + return g_ipv6_loopback_available; +} + #endif /* GRPC_WINSOCK_SOCKET */ diff --git a/src/core/lib/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h index 7bd01eded5a..46d7d583560 100644 --- a/src/core/lib/iomgr/socket_windows.h +++ b/src/core/lib/iomgr/socket_windows.h @@ -92,6 +92,8 @@ typedef struct grpc_winsocket { it will be responsible for closing it. */ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name); +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket); + /* Initiate an asynchronous shutdown of the socket. Will call off any pending operation to cancel them. */ void grpc_winsocket_shutdown(grpc_winsocket* socket); @@ -108,6 +110,10 @@ void grpc_socket_notify_on_read(grpc_winsocket* winsocket, void grpc_socket_become_ready(grpc_winsocket* winsocket, grpc_winsocket_callback_info* ci); +/* Returns true if this system can create AF_INET6 sockets bound to ::1. + The value is probed once, and cached for the life of the process. */ +int grpc_ipv6_loopback_available(void); + #endif #endif /* GRPC_CORE_LIB_IOMGR_SOCKET_WINDOWS_H */ diff --git a/src/core/lib/iomgr/tcp_client_cfstream.cc b/src/core/lib/iomgr/tcp_client_cfstream.cc index 5acea91792f..4b21322d746 100644 --- a/src/core/lib/iomgr/tcp_client_cfstream.cc +++ b/src/core/lib/iomgr/tcp_client_cfstream.cc @@ -211,6 +211,6 @@ static void CFStreamClientConnect(grpc_closure* closure, grpc_endpoint** ep, gpr_mu_unlock(&connect->mu); } -grpc_tcp_client_vtable grpc_posix_tcp_client_vtable = {CFStreamClientConnect}; +grpc_tcp_client_vtable grpc_cfstream_client_vtable = {CFStreamClientConnect}; #endif /* GRPC_CFSTREAM_CLIENT */ diff --git a/src/core/lib/iomgr/tcp_windows.cc b/src/core/lib/iomgr/tcp_windows.cc index 5d316d477b7..b3cb442f18f 100644 --- a/src/core/lib/iomgr/tcp_windows.cc +++ b/src/core/lib/iomgr/tcp_windows.cc @@ -53,7 +53,7 @@ extern grpc_core::TraceFlag grpc_tcp_trace; -static grpc_error* set_non_block(SOCKET sock) { +grpc_error* grpc_tcp_set_non_block(SOCKET sock) { int status; uint32_t param = 1; DWORD ret; @@ -90,7 +90,7 @@ static grpc_error* enable_loopback_fast_path(SOCKET sock) { grpc_error* grpc_tcp_prepare_socket(SOCKET sock) { grpc_error* err; - err = set_non_block(sock); + err = grpc_tcp_set_non_block(sock); if (err != GRPC_ERROR_NONE) return err; err = set_dualstack(sock); if (err != GRPC_ERROR_NONE) return err; diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 161a545a2a5..04ef8102b65 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -46,6 +46,8 @@ grpc_endpoint* grpc_tcp_create(grpc_winsocket* socket, grpc_error* grpc_tcp_prepare_socket(SOCKET sock); +grpc_error* grpc_tcp_set_non_block(SOCKET sock); + #endif #endif /* GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H */ diff --git a/src/core/lib/security/security_connector/load_system_roots.h b/src/core/lib/security/security_connector/load_system_roots.h new file mode 100644 index 00000000000..5fdec154988 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots.h @@ -0,0 +1,29 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H +#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H + +namespace grpc_core { + +// Returns a slice containing roots from the OS trust store +grpc_slice LoadSystemRootCerts(); + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ diff --git a/src/core/lib/security/security_connector/load_system_roots_fallback.cc b/src/core/lib/security/security_connector/load_system_roots_fallback.cc new file mode 100644 index 00000000000..73d1245f331 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_fallback.cc @@ -0,0 +1,32 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include "src/core/lib/security/security_connector/load_system_roots.h" + +#ifndef GPR_LINUX + +namespace grpc_core { + +grpc_slice LoadSystemRootCerts() { return grpc_empty_slice(); } + +} // namespace grpc_core + +#endif /* GPR_LINUX */ diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.cc b/src/core/lib/security/security_connector/load_system_roots_linux.cc new file mode 100644 index 00000000000..924fa8a3e26 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_linux.cc @@ -0,0 +1,165 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include "src/core/lib/security/security_connector/load_system_roots_linux.h" + +#ifdef GPR_LINUX + +#include "src/core/lib/security/security_connector/load_system_roots.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/inlined_vector.h" +#include "src/core/lib/iomgr/load_file.h" + +namespace grpc_core { +namespace { + +const char* kLinuxCertFiles[] = { + "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem", + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}; +const char* kLinuxCertDirectories[] = { + "/etc/ssl/certs", "/system/etc/security/cacerts", "/usr/local/share/certs", + "/etc/pki/tls/certs", "/etc/openssl/certs"}; + +grpc_slice GetSystemRootCerts() { + grpc_slice valid_bundle_slice = grpc_empty_slice(); + size_t num_cert_files_ = GPR_ARRAY_SIZE(kLinuxCertFiles); + for (size_t i = 0; i < num_cert_files_; i++) { + grpc_error* error = + grpc_load_file(kLinuxCertFiles[i], 1, &valid_bundle_slice); + if (error == GRPC_ERROR_NONE) { + return valid_bundle_slice; + } + } + return grpc_empty_slice(); +} + +} // namespace + +void GetAbsoluteFilePath(const char* valid_file_dir, + const char* file_entry_name, char* path_buffer) { + if (valid_file_dir != nullptr && file_entry_name != nullptr) { + int path_len = snprintf(path_buffer, MAXPATHLEN, "%s/%s", valid_file_dir, + file_entry_name); + if (path_len == 0) { + gpr_log(GPR_ERROR, "failed to get absolute path for file: %s", + file_entry_name); + } + } +} + +grpc_slice CreateRootCertsBundle(const char* certs_directory) { + grpc_slice bundle_slice = grpc_empty_slice(); + if (certs_directory == nullptr) { + return bundle_slice; + } + DIR* ca_directory = opendir(certs_directory); + if (ca_directory == nullptr) { + return bundle_slice; + } + struct FileData { + char path[MAXPATHLEN]; + off_t size; + }; + InlinedVector roots_filenames; + size_t total_bundle_size = 0; + struct dirent* directory_entry; + while ((directory_entry = readdir(ca_directory)) != nullptr) { + struct stat dir_entry_stat; + const char* file_entry_name = directory_entry->d_name; + FileData file_data; + GetAbsoluteFilePath(certs_directory, file_entry_name, file_data.path); + int stat_return = stat(file_data.path, &dir_entry_stat); + if (stat_return == -1 || !S_ISREG(dir_entry_stat.st_mode)) { + // no subdirectories. + if (stat_return == -1) { + gpr_log(GPR_ERROR, "failed to get status for file: %s", file_data.path); + } + continue; + } + file_data.size = dir_entry_stat.st_size; + total_bundle_size += file_data.size; + roots_filenames.push_back(file_data); + } + closedir(ca_directory); + char* bundle_string = static_cast(gpr_zalloc(total_bundle_size + 1)); + size_t bytes_read = 0; + for (size_t i = 0; i < roots_filenames.size(); i++) { + int file_descriptor = open(roots_filenames[i].path, O_RDONLY); + if (file_descriptor != -1) { + // Read file into bundle. + size_t cert_file_size = roots_filenames[i].size; + int read_ret = + read(file_descriptor, bundle_string + bytes_read, cert_file_size); + if (read_ret != -1) { + bytes_read += read_ret; + } else { + gpr_log(GPR_ERROR, "failed to read file: %s", roots_filenames[i].path); + } + } + } + bundle_slice = grpc_slice_new(bundle_string, bytes_read, gpr_free); + return bundle_slice; +} + +grpc_slice LoadSystemRootCerts() { + grpc_slice result = grpc_empty_slice(); + // Prioritize user-specified custom directory if flag is set. + char* custom_dir = gpr_getenv("GRPC_SYSTEM_SSL_ROOTS_DIR"); + if (custom_dir != nullptr) { + result = CreateRootCertsBundle(custom_dir); + gpr_free(custom_dir); + } + // If the custom directory is empty/invalid/not specified, fallback to + // distribution-specific directory. + if (GRPC_SLICE_IS_EMPTY(result)) { + result = GetSystemRootCerts(); + } + if (GRPC_SLICE_IS_EMPTY(result)) { + for (size_t i = 0; i < GPR_ARRAY_SIZE(kLinuxCertDirectories); i++) { + result = CreateRootCertsBundle(kLinuxCertDirectories[i]); + if (!GRPC_SLICE_IS_EMPTY(result)) { + break; + } + } + } + return result; +} + +} // namespace grpc_core + +#endif /* GPR_LINUX */ diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.h b/src/core/lib/security/security_connector/load_system_roots_linux.h new file mode 100644 index 00000000000..12617df4928 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_linux.h @@ -0,0 +1,44 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H +#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H + +#include + +#ifdef GPR_LINUX + +namespace grpc_core { + +// Creates a bundle slice containing the contents of all certificate files in +// a directory. +// Returns such slice. +// Exposed for testing purposes only. +grpc_slice CreateRootCertsBundle(const char* certs_directory); + +// Gets the absolute file path needed to load a certificate file. +// Populates path_buffer, which must be of size MAXPATHLEN. +// Exposed for testing purposes only. +void GetAbsoluteFilePath(const char* valid_file_dir, + const char* file_entry_name, char* path_buffer); + +} // namespace grpc_core + +#endif /* GPR_LINUX */ +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H \ + */ diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 59cf3a0af18..04b4c87c713 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -21,7 +21,6 @@ #include "src/core/lib/security/security_connector/security_connector.h" #include -#include #include #include @@ -39,6 +38,7 @@ #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" #include "src/core/lib/security/credentials/ssl/ssl_credentials.h" +#include "src/core/lib/security/security_connector/load_system_roots.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/security/transport/target_authority_table.h" @@ -57,6 +57,12 @@ static const char* installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem"; #endif +/** Environment variable used as a flag to enable/disable loading system root + certificates from the OS trust store. */ +#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#endif + #ifndef TSI_OPENSSL_ALPN_SUPPORT #define TSI_OPENSSL_ALPN_SUPPORT 1 #endif @@ -1186,6 +1192,10 @@ const char* DefaultSslRootStore::GetPemRootCerts() { grpc_slice DefaultSslRootStore::ComputePemRootCerts() { grpc_slice result = grpc_empty_slice(); + char* use_system_roots_env_value = + gpr_getenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + const bool use_system_roots = gpr_is_true(use_system_roots_env_value); + gpr_free(use_system_roots_env_value); // First try to load the roots from the environment. char* default_root_certs_path = gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); @@ -1207,7 +1217,11 @@ grpc_slice DefaultSslRootStore::ComputePemRootCerts() { } gpr_free(pem_root_certs); } - // Fall back to installed certs if needed. + // Try loading roots from OS trust store if flag is enabled. + if (GRPC_SLICE_IS_EMPTY(result) && use_system_roots) { + result = LoadSystemRootCerts(); + } + // Fallback to roots manually shipped with gRPC. if (GRPC_SLICE_IS_EMPTY(result) && ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { GRPC_LOG_IF_ERROR("load_file", diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc index 9b5c6f34906..0f125e7c26d 100644 --- a/src/core/lib/security/transport/client_auth_filter.cc +++ b/src/core/lib/security/transport/client_auth_filter.cc @@ -167,7 +167,6 @@ static void cancel_get_request_metadata(void* arg, grpc_error* error) { grpc_call_credentials_cancel_get_request_metadata( calld->creds, &calld->md_array, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_get_request_metadata"); } static void send_security_metadata(grpc_call_element* elem, @@ -222,7 +221,6 @@ static void send_security_metadata(grpc_call_element* elem, GRPC_ERROR_UNREF(error); } else { // Async return; register cancellation closure with call combiner. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_get_request_metadata"); grpc_call_combiner_set_notify_on_cancel( calld->call_combiner, GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure, @@ -265,7 +263,6 @@ static void cancel_check_call_host(void* arg, grpc_error* error) { chand->security_connector, &calld->async_result_closure, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_check_call_host"); } static void auth_start_transport_stream_op_batch( @@ -318,7 +315,6 @@ static void auth_start_transport_stream_op_batch( GRPC_ERROR_UNREF(error); } else { // Async return; register cancellation closure with call combiner. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_check_call_host"); grpc_call_combiner_set_notify_on_cancel( calld->call_combiner, GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure, diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc index 7732b695b32..5f2ad261d28 100644 --- a/src/core/lib/security/transport/server_auth_filter.cc +++ b/src/core/lib/security/transport/server_auth_filter.cc @@ -160,7 +160,6 @@ static void cancel_call(void* arg, grpc_error* error) { on_md_processing_done_inner(elem, nullptr, 0, nullptr, 0, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_call"); } static void recv_initial_metadata_ready(void* arg, grpc_error* error) { @@ -172,7 +171,6 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) { if (chand->creds != nullptr && chand->creds->processor.process != nullptr) { // We're calling out to the application, so we need to make sure // to drop the call combiner early if we get cancelled. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_call"); GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem, grpc_schedule_on_exec_ctx); grpc_call_combiner_set_notify_on_cancel(calld->call_combiner, diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 151cf6c852a..56c45629529 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -478,6 +478,7 @@ void grpc_call_internal_unref(grpc_call* c REF_ARG) { static void release_call(void* call, grpc_error* error) { grpc_call* c = static_cast(call); grpc_channel* channel = c->channel; + gpr_free(static_cast(const_cast(c->final_info.error_string))); grpc_call_combiner_destroy(&c->call_combiner); grpc_channel_update_call_size_estimate(channel, gpr_arena_destroy(c->arena)); GRPC_CHANNEL_INTERNAL_UNREF(channel, "call"); @@ -520,7 +521,6 @@ static void destroy_call(void* call, grpc_error* error) { grpc_call_stack_destroy(CALL_STACK_FROM_CALL(c), &c->final_info, GRPC_CLOSURE_INIT(&c->release_call, release_call, c, grpc_schedule_on_exec_ctx)); - gpr_free(static_cast(const_cast(c->final_info.error_string))); } void grpc_call_ref(grpc_call* c) { gpr_ref(&c->ext_ref); } @@ -560,8 +560,11 @@ void grpc_call_unref(grpc_call* c) { // Unset the call combiner cancellation closure. This has the // effect of scheduling the previously set cancellation closure, if // any, so that it can release any internal references it may be - // holding to the call stack. + // holding to the call stack. Also flush the closures on exec_ctx so that + // filters that schedule cancel notification closures on exec_ctx do not + // need to take a ref of the call stack to guarantee closure liveness. grpc_call_combiner_set_notify_on_cancel(&c->call_combiner, nullptr); + grpc_core::ExecCtx::Get()->Flush(); } GRPC_CALL_INTERNAL_UNREF(c, "destroy"); } diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 7cbd61adef5..82635d3c215 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -281,6 +281,17 @@ void grpc_channel_get_info(grpc_channel* channel, elem->filter->get_channel_info(elem, channel_info); } +void grpc_channel_reset_connect_backoff(grpc_channel* channel) { + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1, + (channel)); + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->reset_connect_backoff = true; + grpc_channel_element* elem = + grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0); + elem->filter->start_transport_op(elem, op); +} + static grpc_call* grpc_channel_create_call_internal( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, grpc_completion_queue* cq, grpc_pollset_set* pollset_set_alternative, diff --git a/src/core/lib/surface/completion_queue.cc b/src/core/lib/surface/completion_queue.cc index 7da9e6b74ca..0769d9e4f63 100644 --- a/src/core/lib/surface/completion_queue.cc +++ b/src/core/lib/surface/completion_queue.cc @@ -184,7 +184,7 @@ static const cq_poller_vtable g_poller_vtable_by_poller_type[] = { typedef struct cq_vtable { grpc_cq_completion_type cq_completion_type; size_t data_size; - void (*init)(void* data); + void (*init)(void* data, grpc_core::CQCallbackInterface* shutdown_callback); void (*shutdown)(grpc_completion_queue* cq); void (*destroy)(void* data); bool (*begin_op)(grpc_completion_queue* cq, void* tag); @@ -253,6 +253,23 @@ typedef struct cq_pluck_data { plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS]; } cq_pluck_data; +typedef struct cq_callback_data { + /** No actual completed events queue, unlike other types */ + + /** Number of pending events (+1 if we're not shutdown) */ + gpr_atm pending_events; + + /** Counter of how many things have ever been queued on this completion queue + useful for avoiding locks to check the queue */ + gpr_atm things_queued_ever; + + /** 0 initially. 1 once we initiated shutdown */ + bool shutdown_called; + + /** A callback that gets invoked when the CQ completes shutdown */ + grpc_core::CQCallbackInterface* shutdown_callback; +} cq_callback_data; + /* Completion queue structure */ struct grpc_completion_queue { /** Once owning_refs drops to zero, we will destroy the cq */ @@ -276,12 +293,21 @@ struct grpc_completion_queue { /* Forward declarations */ static void cq_finish_shutdown_next(grpc_completion_queue* cq); static void cq_finish_shutdown_pluck(grpc_completion_queue* cq); +static void cq_finish_shutdown_callback(grpc_completion_queue* cq); static void cq_shutdown_next(grpc_completion_queue* cq); static void cq_shutdown_pluck(grpc_completion_queue* cq); +static void cq_shutdown_callback(grpc_completion_queue* cq); static bool cq_begin_op_for_next(grpc_completion_queue* cq, void* tag); static bool cq_begin_op_for_pluck(grpc_completion_queue* cq, void* tag); - +static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* tag); + +// A cq_end_op function is called when an operation on a given CQ with +// a given tag has completed. The storage argument is a reference to the +// space reserved for this completion as it is placed into the corresponding +// queue. The done argument is a callback that will be invoked when it is +// safe to free up that storage. The storage MUST NOT be freed until the +// done callback is invoked. static void cq_end_op_for_next(grpc_completion_queue* cq, void* tag, grpc_error* error, void (*done)(void* done_arg, @@ -294,16 +320,28 @@ static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag, grpc_cq_completion* storage), void* done_arg, grpc_cq_completion* storage); +static void cq_end_op_for_callback(grpc_completion_queue* cq, void* tag, + grpc_error* error, + void (*done)(void* done_arg, + grpc_cq_completion* storage), + void* done_arg, grpc_cq_completion* storage); + static grpc_event cq_next(grpc_completion_queue* cq, gpr_timespec deadline, void* reserved); static grpc_event cq_pluck(grpc_completion_queue* cq, void* tag, gpr_timespec deadline, void* reserved); -static void cq_init_next(void* data); -static void cq_init_pluck(void* data); +// Note that cq_init_next and cq_init_pluck do not use the shutdown_callback +static void cq_init_next(void* data, + grpc_core::CQCallbackInterface* shutdown_callback); +static void cq_init_pluck(void* data, + grpc_core::CQCallbackInterface* shutdown_callback); +static void cq_init_callback(void* data, + grpc_core::CQCallbackInterface* shutdown_callback); static void cq_destroy_next(void* data); static void cq_destroy_pluck(void* data); +static void cq_destroy_callback(void* data); /* Completion queue vtables based on the completion-type */ static const cq_vtable g_cq_vtable[] = { @@ -315,6 +353,10 @@ static const cq_vtable g_cq_vtable[] = { {GRPC_CQ_PLUCK, sizeof(cq_pluck_data), cq_init_pluck, cq_shutdown_pluck, cq_destroy_pluck, cq_begin_op_for_pluck, cq_end_op_for_pluck, nullptr, cq_pluck}, + /* GRPC_CQ_CALLBACK */ + {GRPC_CQ_CALLBACK, sizeof(cq_callback_data), cq_init_callback, + cq_shutdown_callback, cq_destroy_callback, cq_begin_op_for_callback, + cq_end_op_for_callback, nullptr, nullptr}, }; #define DATA_FROM_CQ(cq) ((void*)(cq + 1)) @@ -419,8 +461,8 @@ static long cq_event_queue_num_items(grpc_cq_event_queue* q) { } grpc_completion_queue* grpc_completion_queue_create_internal( - grpc_cq_completion_type completion_type, - grpc_cq_polling_type polling_type) { + grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type, + grpc_core::CQCallbackInterface* shutdown_callback) { GPR_TIMER_SCOPE("grpc_completion_queue_create_internal", 0); grpc_completion_queue* cq; @@ -448,15 +490,16 @@ grpc_completion_queue* grpc_completion_queue_create_internal( gpr_ref_init(&cq->owning_refs, 2); poller_vtable->init(POLLSET_FROM_CQ(cq), &cq->mu); - vtable->init(DATA_FROM_CQ(cq)); + vtable->init(DATA_FROM_CQ(cq), shutdown_callback); GRPC_CLOSURE_INIT(&cq->pollset_shutdown_done, on_pollset_shutdown_done, cq, grpc_schedule_on_exec_ctx); return cq; } -static void cq_init_next(void* ptr) { - cq_next_data* cqd = static_cast(ptr); +static void cq_init_next(void* data, + grpc_core::CQCallbackInterface* shutdown_callback) { + cq_next_data* cqd = static_cast(data); /* Initial count is dropped by grpc_completion_queue_shutdown */ gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->shutdown_called = false; @@ -464,14 +507,15 @@ static void cq_init_next(void* ptr) { cq_event_queue_init(&cqd->queue); } -static void cq_destroy_next(void* ptr) { - cq_next_data* cqd = static_cast(ptr); +static void cq_destroy_next(void* data) { + cq_next_data* cqd = static_cast(data); GPR_ASSERT(cq_event_queue_num_items(&cqd->queue) == 0); cq_event_queue_destroy(&cqd->queue); } -static void cq_init_pluck(void* ptr) { - cq_pluck_data* cqd = static_cast(ptr); +static void cq_init_pluck(void* data, + grpc_core::CQCallbackInterface* shutdown_callback) { + cq_pluck_data* cqd = static_cast(data); /* Initial count is dropped by grpc_completion_queue_shutdown */ gpr_atm_no_barrier_store(&cqd->pending_events, 1); cqd->completed_tail = &cqd->completed_head; @@ -482,11 +526,23 @@ static void cq_init_pluck(void* ptr) { gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); } -static void cq_destroy_pluck(void* ptr) { - cq_pluck_data* cqd = static_cast(ptr); +static void cq_destroy_pluck(void* data) { + cq_pluck_data* cqd = static_cast(data); GPR_ASSERT(cqd->completed_head.next == (uintptr_t)&cqd->completed_head); } +static void cq_init_callback( + void* data, grpc_core::CQCallbackInterface* shutdown_callback) { + cq_callback_data* cqd = static_cast(data); + /* Initial count is dropped by grpc_completion_queue_shutdown */ + gpr_atm_no_barrier_store(&cqd->pending_events, 1); + cqd->shutdown_called = false; + gpr_atm_no_barrier_store(&cqd->things_queued_ever, 0); + cqd->shutdown_callback = shutdown_callback; +} + +static void cq_destroy_callback(void* data) {} + grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue* cq) { return cq->vtable->cq_completion_type; } @@ -596,6 +652,11 @@ static bool cq_begin_op_for_pluck(grpc_completion_queue* cq, void* tag) { return atm_inc_if_nonzero(&cqd->pending_events); } +static bool cq_begin_op_for_callback(grpc_completion_queue* cq, void* tag) { + cq_callback_data* cqd = static_cast DATA_FROM_CQ(cq); + return atm_inc_if_nonzero(&cqd->pending_events); +} + bool grpc_cq_begin_op(grpc_completion_queue* cq, void* tag) { #ifndef NDEBUG gpr_mu_lock(cq->mu); @@ -759,6 +820,48 @@ static void cq_end_op_for_pluck(grpc_completion_queue* cq, void* tag, GRPC_ERROR_UNREF(error); } +/* Complete an event on a completion queue of type GRPC_CQ_CALLBACK */ +static void cq_end_op_for_callback( + grpc_completion_queue* cq, void* tag, grpc_error* error, + void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg, + grpc_cq_completion* storage) { + GPR_TIMER_SCOPE("cq_end_op_for_callback", 0); + + cq_callback_data* cqd = static_cast DATA_FROM_CQ(cq); + bool is_success = (error == GRPC_ERROR_NONE); + + if (grpc_api_trace.enabled() || + (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE)) { + const char* errmsg = grpc_error_string(error); + GRPC_API_TRACE( + "cq_end_op_for_callback(cq=%p, tag=%p, error=%s, " + "done=%p, done_arg=%p, storage=%p)", + 6, (cq, tag, errmsg, done, done_arg, storage)); + if (grpc_trace_operation_failures.enabled() && error != GRPC_ERROR_NONE) { + gpr_log(GPR_ERROR, "Operation failed: tag=%p, error=%s", tag, errmsg); + } + } + + // The callback-based CQ isn't really a queue at all and thus has no need + // for reserved storage. Invoke the done callback right away to release it. + done(done_arg, storage); + + gpr_mu_lock(cq->mu); + cq_check_tag(cq, tag, false); /* Used in debug builds only */ + + gpr_atm_no_barrier_fetch_add(&cqd->things_queued_ever, 1); + if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { + cq_finish_shutdown_callback(cq); + gpr_mu_unlock(cq->mu); + } else { + gpr_mu_unlock(cq->mu); + } + + GRPC_ERROR_UNREF(error); + + (static_cast(tag))->Run(is_success); +} + void grpc_cq_end_op(grpc_completion_queue* cq, void* tag, grpc_error* error, void (*done)(void* done_arg, grpc_cq_completion* storage), void* done_arg, grpc_cq_completion* storage) { @@ -1233,6 +1336,40 @@ static void cq_shutdown_pluck(grpc_completion_queue* cq) { GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (pluck cq)"); } +static void cq_finish_shutdown_callback(grpc_completion_queue* cq) { + cq_callback_data* cqd = static_cast DATA_FROM_CQ(cq); + auto* callback = cqd->shutdown_callback; + + GPR_ASSERT(cqd->shutdown_called); + + cq->poller_vtable->shutdown(POLLSET_FROM_CQ(cq), &cq->pollset_shutdown_done); + callback->Run(true); +} + +static void cq_shutdown_callback(grpc_completion_queue* cq) { + cq_callback_data* cqd = static_cast DATA_FROM_CQ(cq); + + /* Need an extra ref for cq here because: + * We call cq_finish_shutdown_callback() below, which calls pollset shutdown. + * Pollset shutdown decrements the cq ref count which can potentially destroy + * the cq (if that happens to be the last ref). + * Creating an extra ref here prevents the cq from getting destroyed while + * this function is still active */ + GRPC_CQ_INTERNAL_REF(cq, "shutting_down (callback cq)"); + gpr_mu_lock(cq->mu); + if (cqd->shutdown_called) { + gpr_mu_unlock(cq->mu); + GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)"); + return; + } + cqd->shutdown_called = true; + if (gpr_atm_full_fetch_add(&cqd->pending_events, -1) == 1) { + cq_finish_shutdown_callback(cq); + } + gpr_mu_unlock(cq->mu); + GRPC_CQ_INTERNAL_UNREF(cq, "shutting_down (callback cq)"); +} + /* Shutdown simply drops a ref that we reserved at creation time; if we drop to zero here, then enter shutdown mode and wake up any waiters */ void grpc_completion_queue_shutdown(grpc_completion_queue* cq) { diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h index 84446a4d924..a7c524d8e8f 100644 --- a/src/core/lib/surface/completion_queue.h +++ b/src/core/lib/surface/completion_queue.h @@ -25,6 +25,7 @@ #include #include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/abstract.h" #include "src/core/lib/iomgr/pollset.h" /* These trace flags default to 1. The corresponding lines are only traced @@ -47,6 +48,23 @@ typedef struct grpc_cq_completion { uintptr_t next; } grpc_cq_completion; +/// For callback CQs, the tag that is passed in for an operation must +/// actually be a pointer to an implementation of the following class. +/// When the operation completes, the tag will be typecasted from void* +/// to grpc_core::CQCallbackInterface* and then the Run method will be +/// invoked on it. In practice, the language binding (e.g., C++ API +/// implementation) is responsible for providing and using an implementation +/// of this abstract base class. +namespace grpc_core { +class CQCallbackInterface { + public: + virtual ~CQCallbackInterface() {} + virtual void Run(bool) GRPC_ABSTRACT; + + GRPC_ABSTRACT_BASE_CLASS +}; +} // namespace grpc_core + #ifndef NDEBUG void grpc_cq_internal_ref(grpc_completion_queue* cc, const char* reason, const char* file, int line); @@ -87,6 +105,7 @@ grpc_cq_completion_type grpc_get_cq_completion_type(grpc_completion_queue* cc); int grpc_get_cq_poll_num(grpc_completion_queue* cc); grpc_completion_queue* grpc_completion_queue_create_internal( - grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type); + grpc_cq_completion_type completion_type, grpc_cq_polling_type polling_type, + grpc_core::CQCallbackInterface* shutdown_callback); #endif /* GRPC_CORE_LIB_SURFACE_COMPLETION_QUEUE_H */ diff --git a/src/core/lib/surface/completion_queue_factory.cc b/src/core/lib/surface/completion_queue_factory.cc index 51c1183c5f4..ed92dd7eba6 100644 --- a/src/core/lib/surface/completion_queue_factory.cc +++ b/src/core/lib/surface/completion_queue_factory.cc @@ -30,8 +30,9 @@ static grpc_completion_queue* default_create( const grpc_completion_queue_factory* factory, const grpc_completion_queue_attributes* attr) { - return grpc_completion_queue_create_internal(attr->cq_completion_type, - attr->cq_polling_type); + return grpc_completion_queue_create_internal( + attr->cq_completion_type, attr->cq_polling_type, + static_cast(attr->cq_shutdown_cb)); } static grpc_completion_queue_factory_vtable default_vtable = {default_create}; @@ -60,14 +61,22 @@ const grpc_completion_queue_factory* grpc_completion_queue_factory_lookup( grpc_completion_queue* grpc_completion_queue_create_for_next(void* reserved) { GPR_ASSERT(!reserved); grpc_completion_queue_attributes attr = {1, GRPC_CQ_NEXT, - GRPC_CQ_DEFAULT_POLLING}; + GRPC_CQ_DEFAULT_POLLING, nullptr}; return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr); } grpc_completion_queue* grpc_completion_queue_create_for_pluck(void* reserved) { GPR_ASSERT(!reserved); grpc_completion_queue_attributes attr = {1, GRPC_CQ_PLUCK, - GRPC_CQ_DEFAULT_POLLING}; + GRPC_CQ_DEFAULT_POLLING, nullptr}; + return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr); +} + +grpc_completion_queue* grpc_completion_queue_create_for_callback( + void* shutdown_callback, void* reserved) { + GPR_ASSERT(!reserved); + grpc_completion_queue_attributes attr = { + 2, GRPC_CQ_CALLBACK, GRPC_CQ_DEFAULT_POLLING, shutdown_callback}; return g_default_cq_factory.vtable->create(&g_default_cq_factory, &attr); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 585b9dfae91..9e784635c69 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -282,6 +282,8 @@ typedef struct grpc_transport_op { /** Called when the ping ack is received */ grpc_closure* on_ack; } send_ping; + // If true, will reset the channel's connection backoff. + bool reset_connect_backoff; /*************************************************************************** * remaining fields are initialized and used at the discretion of the diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 06b999899ac..1df1021bb17 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -462,6 +462,14 @@ void alts_tsi_handshaker_handle_response(alts_tsi_handshaker* handshaker, set_unused_bytes(result, &handshaker->recv_bytes, resp->bytes_consumed); } grpc_status_code code = static_cast(resp->status.code); + if (code != GRPC_STATUS_OK) { + grpc_slice* details = static_cast(resp->status.details.arg); + if (details != nullptr) { + char* error_details = grpc_slice_to_c_string(*details); + gpr_log(GPR_ERROR, "Error from handshaker service:%s", error_details); + gpr_free(error_details); + } + } grpc_gcp_handshaker_resp_destroy(resp); cb(alts_tsi_utils_convert_to_tsi_result(code), user_data, bytes_to_send, bytes_to_send_size, result); diff --git a/src/core/tsi/ssl/session_cache/ssl_session_cache.cc b/src/core/tsi/ssl/session_cache/ssl_session_cache.cc index fe4f83a13de..ce74fde3434 100644 --- a/src/core/tsi/ssl/session_cache/ssl_session_cache.cc +++ b/src/core/tsi/ssl/session_cache/ssl_session_cache.cc @@ -18,9 +18,9 @@ #include -#include "src/core/tsi/ssl/session_cache/ssl_session_cache.h" - +#include "src/core/lib/gprpp/mutex_lock.h" #include "src/core/tsi/ssl/session_cache/ssl_session.h" +#include "src/core/tsi/ssl/session_cache/ssl_session_cache.h" #include #include @@ -97,7 +97,7 @@ SslSessionLRUCache::~SslSessionLRUCache() { } size_t SslSessionLRUCache::Size() { - grpc_core::mu_guard guard(&lock_); + grpc_core::MutexLock lock(&lock_); return use_order_list_size_; } @@ -117,7 +117,7 @@ SslSessionLRUCache::Node* SslSessionLRUCache::FindLocked( } void SslSessionLRUCache::Put(const char* key, SslSessionPtr session) { - grpc_core::mu_guard guard(&lock_); + grpc_core::MutexLock lock(&lock_); Node* node = FindLocked(grpc_slice_from_static_string(key)); if (node != nullptr) { node->SetSession(std::move(session)); @@ -140,7 +140,7 @@ void SslSessionLRUCache::Put(const char* key, SslSessionPtr session) { } SslSessionPtr SslSessionLRUCache::Get(const char* key) { - grpc_core::mu_guard guard(&lock_); + grpc_core::MutexLock lock(&lock_); // Key is only used for lookups. grpc_slice key_slice = grpc_slice_from_static_string(key); Node* node = FindLocked(key_slice); diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc index 867f31f0255..39b891c2e1f 100644 --- a/src/cpp/client/channel_cc.cc +++ b/src/cpp/client/channel_cc.cc @@ -84,6 +84,14 @@ grpc::string Channel::GetServiceConfigJSON() const { &channel_info.service_config_json); } +namespace experimental { + +void ChannelResetConnectionBackoff(Channel* channel) { + grpc_channel_reset_connect_backoff(channel->c_channel_); +} + +} // namespace experimental + internal::Call Channel::CreateCall(const internal::RpcMethod& method, ClientContext* context, CompletionQueue* cq) { diff --git a/src/cpp/client/secure_credentials.cc b/src/cpp/client/secure_credentials.cc index bdb63596326..e48fbeb86d7 100644 --- a/src/cpp/client/secure_credentials.cc +++ b/src/cpp/client/secure_credentials.cc @@ -107,6 +107,13 @@ std::shared_ptr AltsCredentials( return WrapChannelCredentials(c_creds); } +// Builds Local Credentials +std::shared_ptr LocalCredentials( + grpc_local_connect_type type) { + GrpcLibraryCodegen init; // To call grpc_init(). + return WrapChannelCredentials(grpc_local_credentials_create(type)); +} + } // namespace experimental // Builds credentials for use when running in GCE diff --git a/src/cpp/common/resource_quota_cc.cc b/src/cpp/common/resource_quota_cc.cc index daeb0ba171c..276e5f79548 100644 --- a/src/cpp/common/resource_quota_cc.cc +++ b/src/cpp/common/resource_quota_cc.cc @@ -33,4 +33,8 @@ ResourceQuota& ResourceQuota::Resize(size_t new_size) { return *this; } +ResourceQuota& ResourceQuota::SetMaxThreads(int new_max_threads) { + grpc_resource_quota_set_max_threads(impl_, new_max_threads); + return *this; +} } // namespace grpc diff --git a/src/cpp/ext/filters/census/grpc_plugin.h b/src/cpp/ext/filters/census/grpc_plugin.h index 7ff2e7a8b8c..9e319cb994e 100644 --- a/src/cpp/ext/filters/census/grpc_plugin.h +++ b/src/cpp/ext/filters/census/grpc_plugin.h @@ -24,15 +24,11 @@ #include "absl/strings/string_view.h" #include "include/grpcpp/opencensus.h" #include "opencensus/stats/stats.h" -#include "opencensus/trace/span.h" namespace grpc { class ServerContext; -// Returns the tracing Span for the current RPC. -::opencensus::trace::Span GetSpanFromServerContext(ServerContext* context); - // The tag keys set when recording RPC stats. ::opencensus::stats::TagKey ClientMethodTagKey(); ::opencensus::stats::TagKey ClientStatusTagKey(); diff --git a/src/cpp/server/load_reporter/util.cc b/src/cpp/server/load_reporter/util.cc index a2f2f11e707..89bdf57049c 100644 --- a/src/cpp/server/load_reporter/util.cc +++ b/src/cpp/server/load_reporter/util.cc @@ -20,6 +20,8 @@ #include +#include + #include namespace grpc { diff --git a/src/cpp/server/secure_server_credentials.cc b/src/cpp/server/secure_server_credentials.cc index a5af25751af..536bf022dde 100644 --- a/src/cpp/server/secure_server_credentials.cc +++ b/src/cpp/server/secure_server_credentials.cc @@ -139,5 +139,11 @@ std::shared_ptr AltsServerCredentials( new SecureServerCredentials(c_creds)); } +std::shared_ptr LocalServerCredentials( + grpc_local_connect_type type) { + return std::shared_ptr( + new SecureServerCredentials(grpc_local_server_credentials_create(type))); +} + } // namespace experimental } // namespace grpc diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc index e0b9b7a62bd..8417c45e641 100644 --- a/src/cpp/server/server_builder.cc +++ b/src/cpp/server/server_builder.cc @@ -263,7 +263,7 @@ std::unique_ptr ServerBuilder::BuildAndStart() { std::unique_ptr server(new Server( max_receive_message_size_, &args, sync_server_cqs, sync_server_settings_.min_pollers, sync_server_settings_.max_pollers, - sync_server_settings_.cq_timeout_msec)); + sync_server_settings_.cq_timeout_msec, resource_quota_)); if (has_sync_methods) { // This is a Sync server diff --git a/src/cpp/server/server_cc.cc b/src/cpp/server/server_cc.cc index 0d77510e29d..36c709eb45b 100644 --- a/src/cpp/server/server_cc.cc +++ b/src/cpp/server/server_cc.cc @@ -47,6 +47,13 @@ namespace grpc { namespace { +// The default value for maximum number of threads that can be created in the +// sync server. This value of INT_MAX is chosen to match the default behavior if +// no ResourceQuota is set. To modify the max number of threads in a sync +// server, pass a custom ResourceQuota object (with the desired number of +// max-threads set) to the server builder. +#define DEFAULT_MAX_SYNC_SERVER_THREADS INT_MAX + class DefaultGlobalCallbacks final : public Server::GlobalCallbacks { public: ~DefaultGlobalCallbacks() override {} @@ -204,8 +211,10 @@ class Server::SyncRequest final : public internal::CompletionQueueTag { call_(mrd->call_, server, &cq_, server->max_receive_message_size()), ctx_(mrd->deadline_, &mrd->request_metadata_), has_request_payload_(mrd->has_request_payload_), - request_payload_(mrd->request_payload_), - method_(mrd->method_) { + request_payload_(has_request_payload_ ? mrd->request_payload_ + : nullptr), + method_(mrd->method_), + server_(server) { ctx_.set_call(mrd->call_); ctx_.cq_ = &cq_; GPR_ASSERT(mrd->in_flight_); @@ -219,10 +228,13 @@ class Server::SyncRequest final : public internal::CompletionQueueTag { } } - void Run(const std::shared_ptr& global_callbacks) { + void Run(const std::shared_ptr& global_callbacks, + bool resources) { ctx_.BeginCompletionOp(&call_); global_callbacks->PreSynchronousRequest(&ctx_); - method_->handler()->RunHandler(internal::MethodHandler::HandlerParameter( + auto* handler = resources ? method_->handler() + : server_->resource_exhausted_handler_.get(); + handler->RunHandler(internal::MethodHandler::HandlerParameter( &call_, &ctx_, request_payload_)); global_callbacks->PostSynchronousRequest(&ctx_); request_payload_ = nullptr; @@ -244,6 +256,7 @@ class Server::SyncRequest final : public internal::CompletionQueueTag { const bool has_request_payload_; grpc_byte_buffer* request_payload_; internal::RpcServiceMethod* const method_; + Server* server_; }; private: @@ -266,9 +279,9 @@ class Server::SyncRequestThreadManager : public ThreadManager { public: SyncRequestThreadManager(Server* server, CompletionQueue* server_cq, std::shared_ptr global_callbacks, - int min_pollers, int max_pollers, - int cq_timeout_msec) - : ThreadManager(min_pollers, max_pollers), + grpc_resource_quota* rq, int min_pollers, + int max_pollers, int cq_timeout_msec) + : ThreadManager("SyncServer", rq, min_pollers, max_pollers), server_(server), server_cq_(server_cq), cq_timeout_msec_(cq_timeout_msec), @@ -294,7 +307,7 @@ class Server::SyncRequestThreadManager : public ThreadManager { GPR_UNREACHABLE_CODE(return TIMEOUT); } - void DoWork(void* tag, bool ok) override { + void DoWork(void* tag, bool ok, bool resources) override { SyncRequest* sync_req = static_cast(tag); if (!sync_req) { @@ -314,7 +327,7 @@ class Server::SyncRequestThreadManager : public ThreadManager { } GPR_TIMER_SCOPE("cd.Run()", 0); - cd.Run(global_callbacks_); + cd.Run(global_callbacks_, resources); } // TODO (sreek) If ok is false here (which it isn't in case of // grpc_request_registered_call), we should still re-queue the request @@ -376,7 +389,8 @@ Server::Server( int max_receive_message_size, ChannelArguments* args, std::shared_ptr>> sync_server_cqs, - int min_pollers, int max_pollers, int sync_cq_timeout_msec) + int min_pollers, int max_pollers, int sync_cq_timeout_msec, + grpc_resource_quota* server_rq) : max_receive_message_size_(max_receive_message_size), sync_server_cqs_(std::move(sync_server_cqs)), started_(false), @@ -392,10 +406,22 @@ Server::Server( global_callbacks_->UpdateArguments(args); if (sync_server_cqs_ != nullptr) { + bool default_rq_created = false; + if (server_rq == nullptr) { + server_rq = grpc_resource_quota_create("SyncServer-default-rq"); + grpc_resource_quota_set_max_threads(server_rq, + DEFAULT_MAX_SYNC_SERVER_THREADS); + default_rq_created = true; + } + for (const auto& it : *sync_server_cqs_) { sync_req_mgrs_.emplace_back(new SyncRequestThreadManager( - this, it.get(), global_callbacks_, min_pollers, max_pollers, - sync_cq_timeout_msec)); + this, it.get(), global_callbacks_, server_rq, min_pollers, + max_pollers, sync_cq_timeout_msec)); + } + + if (default_rq_created) { + grpc_resource_quota_unref(server_rq); } } @@ -559,6 +585,13 @@ void Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) { } } + // If this server has any support for synchronous methods (has any sync + // server CQs), make sure that we have a ResourceExhausted handler + // to deal with the case of thread exhaustion + if (sync_server_cqs_ != nullptr && !sync_server_cqs_->empty()) { + resource_exhausted_handler_.reset(new internal::ResourceExhaustedHandler); + } + for (auto it = sync_req_mgrs_.begin(); it != sync_req_mgrs_.end(); it++) { (*it)->Start(); } diff --git a/src/cpp/thread_manager/thread_manager.cc b/src/cpp/thread_manager/thread_manager.cc index 02ac56a3fdc..3e8606a76fd 100644 --- a/src/cpp/thread_manager/thread_manager.cc +++ b/src/cpp/thread_manager/thread_manager.cc @@ -22,8 +22,8 @@ #include #include - #include "src/core/lib/gprpp/thd.h" +#include "src/core/lib/iomgr/exec_ctx.h" namespace grpc { @@ -48,12 +48,17 @@ ThreadManager::WorkerThread::~WorkerThread() { thd_.Join(); } -ThreadManager::ThreadManager(int min_pollers, int max_pollers) +ThreadManager::ThreadManager(const char* name, + grpc_resource_quota* resource_quota, + int min_pollers, int max_pollers) : shutdown_(false), num_pollers_(0), min_pollers_(min_pollers), max_pollers_(max_pollers == -1 ? INT_MAX : max_pollers), - num_threads_(0) {} + num_threads_(0), + max_active_threads_sofar_(0) { + resource_user_ = grpc_resource_user_create(resource_quota, name); +} ThreadManager::~ThreadManager() { { @@ -61,6 +66,8 @@ ThreadManager::~ThreadManager() { GPR_ASSERT(num_threads_ == 0); } + grpc_core::ExecCtx exec_ctx; // grpc_resource_user_unref needs an exec_ctx + grpc_resource_user_unref(resource_user_); CleanupCompletedThreads(); } @@ -81,17 +88,27 @@ bool ThreadManager::IsShutdown() { return shutdown_; } +int ThreadManager::GetMaxActiveThreadsSoFar() { + std::lock_guard list_lock(list_mu_); + return max_active_threads_sofar_; +} + void ThreadManager::MarkAsCompleted(WorkerThread* thd) { { std::lock_guard list_lock(list_mu_); completed_threads_.push_back(thd); } - std::lock_guard lock(mu_); - num_threads_--; - if (num_threads_ == 0) { - shutdown_cv_.notify_one(); + { + std::lock_guard lock(mu_); + num_threads_--; + if (num_threads_ == 0) { + shutdown_cv_.notify_one(); + } } + + // Give a thread back to the resource quota + grpc_resource_user_free_threads(resource_user_, 1); } void ThreadManager::CleanupCompletedThreads() { @@ -106,14 +123,22 @@ void ThreadManager::CleanupCompletedThreads() { } void ThreadManager::Initialize() { + if (!grpc_resource_user_allocate_threads(resource_user_, min_pollers_)) { + gpr_log(GPR_ERROR, + "No thread quota available to even create the minimum required " + "polling threads (i.e %d). Unable to start the thread manager", + min_pollers_); + abort(); + } + { std::unique_lock lock(mu_); num_pollers_ = min_pollers_; num_threads_ = min_pollers_; + max_active_threads_sofar_ = min_pollers_; } for (int i = 0; i < min_pollers_; i++) { - // Create a new thread (which ends up calling the MainWorkLoop() function new WorkerThread(this); } } @@ -139,20 +164,40 @@ void ThreadManager::MainWorkLoop() { done = true; break; case WORK_FOUND: - // If we got work and there are now insufficient pollers, start a new - // one + // If we got work and there are now insufficient pollers and there is + // quota available to create a new thread, start a new poller thread + bool resource_exhausted = false; if (!shutdown_ && num_pollers_ < min_pollers_) { - num_pollers_++; - num_threads_++; - // Drop lock before spawning thread to avoid contention - lock.unlock(); - new WorkerThread(this); + if (grpc_resource_user_allocate_threads(resource_user_, 1)) { + // We can allocate a new poller thread + num_pollers_++; + num_threads_++; + if (num_threads_ > max_active_threads_sofar_) { + max_active_threads_sofar_ = num_threads_; + } + // Drop lock before spawning thread to avoid contention + lock.unlock(); + new WorkerThread(this); + } else if (num_pollers_ > 0) { + // There is still at least some thread polling, so we can go on + // even though we are below the number of pollers that we would + // like to have (min_pollers_) + lock.unlock(); + } else { + // There are no pollers to spare and we couldn't allocate + // a new thread, so resources are exhausted! + lock.unlock(); + resource_exhausted = true; + } } else { - // Drop lock for consistency with above branch + // There are a sufficient number of pollers available so we can do + // the work and continue polling with our existing poller threads lock.unlock(); } // Lock is always released at this point - do the application work - DoWork(tag, ok); + // or return resource exhausted if there is new work but we couldn't + // get a thread in which to do it. + DoWork(tag, ok, !resource_exhausted); // Take the lock again to check post conditions lock.lock(); // If we're shutdown, we should finish at this point. @@ -196,6 +241,8 @@ void ThreadManager::MainWorkLoop() { } }; + // This thread is exiting. Do some cleanup work i.e delete already completed + // worker threads CleanupCompletedThreads(); // If we are here, either ThreadManager is shutting down or it already has diff --git a/src/cpp/thread_manager/thread_manager.h b/src/cpp/thread_manager/thread_manager.h index 5a40f2de477..6f0bd17c5fe 100644 --- a/src/cpp/thread_manager/thread_manager.h +++ b/src/cpp/thread_manager/thread_manager.h @@ -27,12 +27,14 @@ #include #include "src/core/lib/gprpp/thd.h" +#include "src/core/lib/iomgr/resource_quota.h" namespace grpc { class ThreadManager { public: - explicit ThreadManager(int min_pollers, int max_pollers); + explicit ThreadManager(const char* name, grpc_resource_quota* resource_quota, + int min_pollers, int max_pollers); virtual ~ThreadManager(); // Initializes and Starts the Rpc Manager threads @@ -65,12 +67,14 @@ class ThreadManager { // The implementation of DoWork() is supposed to perform the work found by // PollForWork(). The tag and ok parameters are the same as returned by - // PollForWork() + // PollForWork(). The resources parameter indicates that the call actually + // has the resources available for performing the RPC's work. If it doesn't, + // the implementation should fail it appropriately. // // The implementation of DoWork() should also do any setup needed to ensure // that the next call to PollForWork() (not necessarily by the current thread) // actually finds some work - virtual void DoWork(void* tag, bool ok) = 0; + virtual void DoWork(void* tag, bool ok, bool resources) = 0; // Mark the ThreadManager as shutdown and begin draining the work. This is a // non-blocking call and the caller should call Wait(), a blocking call which @@ -84,6 +88,11 @@ class ThreadManager { // all the threads have drained all the outstanding work virtual void Wait(); + // Max number of concurrent threads that were ever active in this thread + // manager so far. This is useful for debugging purposes (and in unit tests) + // to check if resource_quota is properly being enforced. + int GetMaxActiveThreadsSoFar(); + private: // Helper wrapper class around grpc_core::Thread. Takes a ThreadManager object // and starts a new grpc_core::Thread to calls the Run() function. @@ -91,6 +100,24 @@ class ThreadManager { // The Run() function calls ThreadManager::MainWorkLoop() function and once // that completes, it marks the WorkerThread completed by calling // ThreadManager::MarkAsCompleted() + // + // WHY IS THIS NEEDED?: + // When a thread terminates, some other thread *must* call Join() on that + // thread so that the resources are released. Having a WorkerThread wrapper + // will make this easier. Once Run() completes, each thread calls the + // following two functions: + // ThreadManager::CleanupCompletedThreads() + // ThreadManager::MarkAsCompleted() + // + // - MarkAsCompleted() puts the WorkerThread object in the ThreadManger's + // completed_threads_ list + // - CleanupCompletedThreads() calls "Join()" on the threads that are already + // in the completed_threads_ list (since a thread cannot call Join() on + // itself, it calls CleanupCompletedThreads() *before* calling + // MarkAsCompleted()) + // + // TODO(sreek): Consider creating the threads 'detached' so that Join() need + // not be called (and the need for this WorkerThread class is eliminated) class WorkerThread { public: WorkerThread(ThreadManager* thd_mgr); @@ -111,13 +138,21 @@ class ThreadManager { void MarkAsCompleted(WorkerThread* thd); void CleanupCompletedThreads(); - // Protects shutdown_, num_pollers_ and num_threads_ - // TODO: sreek - Change num_pollers and num_threads_ to atomics + // Protects shutdown_, num_pollers_, num_threads_ and + // max_active_threads_sofar_ std::mutex mu_; bool shutdown_; std::condition_variable shutdown_cv_; + // The resource user object to use when requesting quota to create threads + // + // Note: The user of this ThreadManager object must create grpc_resource_quota + // object (that contains the actual max thread quota) and a grpc_resource_user + // object through which quota is requested whenver new threads need to be + // created + grpc_resource_user* resource_user_; + // Number of threads doing polling int num_pollers_; @@ -125,10 +160,15 @@ class ThreadManager { int min_pollers_; int max_pollers_; - // The total number of threads (includes threads includes the threads that are - // currently polling i.e num_pollers_) + // The total number of threads currently active (includes threads includes the + // threads that are currently polling i.e num_pollers_) int num_threads_; + // See GetMaxActiveThreadsSoFar()'s description. + // To be more specific, this variable tracks the max value num_threads_ was + // ever set so far + int max_active_threads_sofar_; + std::mutex list_mu_; std::list completed_threads_; }; diff --git a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj index 18993a93e00..d58f0468244 100755 --- a/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj +++ b/src/csharp/Grpc.Core.Tests/Grpc.Core.Tests.csproj @@ -17,8 +17,8 @@ - - + + diff --git a/src/csharp/Grpc.Core.Tests/SanityTest.cs b/src/csharp/Grpc.Core.Tests/SanityTest.cs index 73efad1f842..eaad409ec0b 100644 --- a/src/csharp/Grpc.Core.Tests/SanityTest.cs +++ b/src/csharp/Grpc.Core.Tests/SanityTest.cs @@ -65,13 +65,13 @@ namespace Grpc.Core.Tests { foreach (var m in t.GetMethods()) { - var attributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true); - if (attributes.Length > 0) + var testAttributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestAttribute), true); + var testCaseAttributes = m.GetCustomAttributes(typeof(NUnit.Framework.TestCaseAttribute), true); + if (testAttributes.Length > 0 || testCaseAttributes.Length > 0) { testClasses.Add(t.FullName); break; } - } } testClasses.Sort(); diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index 0da95d203c6..dc5683c9753 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -47,19 +47,35 @@ true - runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so + native/android/armeabi-v7a/libgrpc_csharp_ext.so true - runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so + native/android/arm64-v8a/libgrpc_csharp_ext.so + true + + + native/android/x86/libgrpc_csharp_ext.so + true + + + native/ios/universal/libgrpc_csharp_ext.a + true + + + native/ios/universal/libgrpc.a true build/net45/ true - - build/MonoAndroid/ + + build/MonoAndroid10/ + true + + + build/Xamarin.iOS10/ true diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index a6a1d8af50e..6ca694e0e46 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -50,6 +50,7 @@ namespace Grpc.Core static int requestCallContextPoolThreadLocalCapacity = DefaultRequestCallContextPoolThreadLocalCapacity; static readonly HashSet registeredChannels = new HashSet(); static readonly HashSet registeredServers = new HashSet(); + static readonly AtomicCounter nativeInitCounter = new AtomicCounter(); static ILogger logger = new LogLevelFilterLogger(new ConsoleLogger(), LogLevel.Off, true); @@ -360,12 +361,25 @@ namespace Grpc.Core internal static void GrpcNativeInit() { + if (!IsNativeShutdownAllowed && nativeInitCounter.Count > 0) + { + // Normally grpc_init and grpc_shutdown calls should come in pairs (C core does reference counting), + // but in case we avoid grpc_shutdown calls altogether, calling grpc_init has no effect + // besides incrementing an internal C core counter that could theoretically overflow. + // To avoid this theoretical possibility we guard repeated calls to grpc_init() + // with a 64-bit atomic counter (that can't realistically overflow). + return; + } NativeMethods.Get().grpcsharp_init(); + nativeInitCounter.Increment(); } internal static void GrpcNativeShutdown() { - NativeMethods.Get().grpcsharp_shutdown(); + if (IsNativeShutdownAllowed) + { + NativeMethods.Get().grpcsharp_shutdown(); + } } /// @@ -411,6 +425,14 @@ namespace Grpc.Core return GetThreadPoolSizeOrDefault(); } + // On some platforms (specifically iOS), thread local variables in native code + // require initialization/destruction. By skipping the grpc_shutdown() call, + // we avoid a potential crash where grpc_shutdown() has already destroyed + // the thread local variables, but some C core's *_destroy() methods still + // need to run (e.g. they may be run by finalizer thread which is out of our control) + // For more context, see https://github.com/grpc/grpc/issues/16294 + private static bool IsNativeShutdownAllowed => !PlatformApis.IsXamarinIOS && !PlatformApis.IsUnityIOS; + private static class ShutdownHooks { static object staticLock = new object(); diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index c501aa89fb6..a8f147545b4 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -42,6 +42,7 @@ namespace Grpc.Core.Internal static readonly bool isMono; static readonly bool isNetCore; static readonly bool isUnity; + static readonly bool isUnityIOS; static readonly bool isXamarin; static readonly bool isXamarinIOS; static readonly bool isXamarinAndroid; @@ -63,7 +64,25 @@ namespace Grpc.Core.Internal isNetCore = false; #endif isMono = Type.GetType("Mono.Runtime") != null; - isUnity = Type.GetType(UnityEngineApplicationClassName) != null; + + // Unity + var unityApplicationClass = Type.GetType(UnityEngineApplicationClassName); + if (unityApplicationClass != null) + { + isUnity = true; + // Consult value of Application.platform via reflection + // https://docs.unity3d.com/ScriptReference/Application-platform.html + var platformProperty = unityApplicationClass.GetTypeInfo().GetProperty("platform"); + var unityRuntimePlatform = platformProperty?.GetValue(null)?.ToString(); + isUnityIOS = (unityRuntimePlatform == "IPhonePlayer"); + } + else + { + isUnity = false; + isUnityIOS = false; + } + + // Xamarin isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null; isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null; isXamarin = isXamarinIOS || isXamarinAndroid; @@ -97,6 +116,14 @@ namespace Grpc.Core.Internal get { return isUnity; } } + /// + /// true if running on Unity iOS, false otherwise. + /// + public static bool IsUnityIOS + { + get { return isUnityIOS; } + } + /// /// true if running on a Xamarin platform (either Xamarin.Android or Xamarin.iOS), /// false otherwise. diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 1a746acdd4f..45bd8ebd855 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -2,6 +2,6 @@ 1.15.0-dev - 3.5.1 + 3.6.1 diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets deleted file mode 100644 index f764f4cae1d..00000000000 --- a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets +++ /dev/null @@ -1,21 +0,0 @@ - - - - <_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\ - - - - - Always - arm64-v8a - - - - - - Always - armeabi-v7a - - - - diff --git a/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets new file mode 100644 index 00000000000..250d3bd0cd6 --- /dev/null +++ b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets @@ -0,0 +1,25 @@ + + + + + + Always + arm64-v8a + + + + + + Always + armeabi-v7a + + + + + + Always + x86 + + + + diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets new file mode 100644 index 00000000000..dda1cdd1e84 --- /dev/null +++ b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets @@ -0,0 +1,15 @@ + + + + + + Static + True + + + Static + True + + + + diff --git a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj index d2cc5bbc65e..7493eb8051d 100755 --- a/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj +++ b/src/csharp/Grpc.Examples.Tests/Grpc.Examples.Tests.csproj @@ -17,8 +17,8 @@ - - + + diff --git a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj index 9da0539dcb9..616e56df105 100755 --- a/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj +++ b/src/csharp/Grpc.HealthCheck.Tests/Grpc.HealthCheck.Tests.csproj @@ -16,8 +16,8 @@ - - + + diff --git a/src/csharp/Grpc.IntegrationTesting/Control.cs b/src/csharp/Grpc.IntegrationTesting/Control.cs index f3284a5390f..6e003484513 100644 --- a/src/csharp/Grpc.IntegrationTesting/Control.cs +++ b/src/csharp/Grpc.IntegrationTesting/Control.cs @@ -1039,14 +1039,14 @@ namespace Grpc.Testing { public ClientConfig(ClientConfig other) : this() { serverTargets_ = other.serverTargets_.Clone(); clientType_ = other.clientType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; outstandingRpcsPerChannel_ = other.outstandingRpcsPerChannel_; clientChannels_ = other.clientChannels_; asyncClientThreads_ = other.asyncClientThreads_; rpcType_ = other.rpcType_; - LoadParams = other.loadParams_ != null ? other.LoadParams.Clone() : null; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; - HistogramParams = other.histogramParams_ != null ? other.HistogramParams.Clone() : null; + loadParams_ = other.loadParams_ != null ? other.loadParams_.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; + histogramParams_ = other.histogramParams_ != null ? other.histogramParams_.Clone() : null; coreList_ = other.coreList_.Clone(); coreLimit_ = other.coreLimit_; otherClientApi_ = other.otherClientApi_; @@ -1647,7 +1647,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStatus(ClientStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -2122,11 +2122,11 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerConfig(ServerConfig other) : this() { serverType_ = other.serverType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; port_ = other.port_; asyncServerThreads_ = other.asyncServerThreads_; coreLimit_ = other.coreLimit_; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; coreList_ = other.coreList_.Clone(); otherServerApi_ = other.otherServerApi_; threadsPerCq_ = other.threadsPerCq_; @@ -2758,7 +2758,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerStatus(ServerStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; port_ = other.port_; cores_ = other.cores_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -3293,9 +3293,9 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public Scenario(Scenario other) : this() { name_ = other.name_; - ClientConfig = other.clientConfig_ != null ? other.ClientConfig.Clone() : null; + clientConfig_ = other.clientConfig_ != null ? other.clientConfig_.Clone() : null; numClients_ = other.numClients_; - ServerConfig = other.serverConfig_ != null ? other.ServerConfig.Clone() : null; + serverConfig_ = other.serverConfig_ != null ? other.serverConfig_.Clone() : null; numServers_ = other.numServers_; warmupSeconds_ = other.warmupSeconds_; benchmarkSeconds_ = other.benchmarkSeconds_; @@ -4422,12 +4422,12 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ScenarioResult(ScenarioResult other) : this() { - Scenario = other.scenario_ != null ? other.Scenario.Clone() : null; - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + scenario_ = other.scenario_ != null ? other.scenario_.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; clientStats_ = other.clientStats_.Clone(); serverStats_ = other.serverStats_.Clone(); serverCores_ = other.serverCores_.Clone(); - Summary = other.summary_ != null ? other.Summary.Clone() : null; + summary_ = other.summary_ != null ? other.summary_.Clone() : null; clientSuccess_ = other.clientSuccess_.Clone(); serverSuccess_ = other.serverSuccess_.Clone(); requestResults_ = other.requestResults_.Clone(); diff --git a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs index 39c3d76ce82..80a1007e9a5 100644 --- a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs +++ b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs @@ -28,7 +28,7 @@ namespace Grpc.Testing { "DGdycGMudGVzdGluZyIyCglEZWJ1Z0luZm8SFQoNc3RhY2tfZW50cmllcxgB", "IAMoCRIOCgZkZXRhaWwYAiABKAkiUAoLRXJyb3JTdGF0dXMSDAoEY29kZRgB", "IAEoBRIVCg1lcnJvcl9tZXNzYWdlGAIgASgJEhwKFGJpbmFyeV9lcnJvcl9k", - "ZXRhaWxzGAMgASgJIuIDCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", + "ZXRhaWxzGAMgASgJIv8DCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", "bmUYASABKAgSHgoWY2xpZW50X2NhbmNlbF9hZnRlcl91cxgCIAEoBRIeChZz", "ZXJ2ZXJfY2FuY2VsX2FmdGVyX3VzGAMgASgFEhUKDWVjaG9fbWV0YWRhdGEY", "BCABKAgSGgoSY2hlY2tfYXV0aF9jb250ZXh0GAUgASgIEh8KF3Jlc3BvbnNl", @@ -39,18 +39,18 @@ namespace Grpc.Testing { "Zy5EZWJ1Z0luZm8SEgoKc2VydmVyX2RpZRgMIAEoCBIcChRiaW5hcnlfZXJy", "b3JfZGV0YWlscxgNIAEoCRIxCg5leHBlY3RlZF9lcnJvchgOIAEoCzIZLmdy", "cGMudGVzdGluZy5FcnJvclN0YXR1cxIXCg9zZXJ2ZXJfc2xlZXBfdXMYDyAB", - "KAUiSgoLRWNob1JlcXVlc3QSDwoHbWVzc2FnZRgBIAEoCRIqCgVwYXJhbRgC", - "IAEoCzIbLmdycGMudGVzdGluZy5SZXF1ZXN0UGFyYW1zIkYKDlJlc3BvbnNl", - "UGFyYW1zEhgKEHJlcXVlc3RfZGVhZGxpbmUYASABKAMSDAoEaG9zdBgCIAEo", - "CRIMCgRwZWVyGAMgASgJIkwKDEVjaG9SZXNwb25zZRIPCgdtZXNzYWdlGAEg", - "ASgJEisKBXBhcmFtGAIgASgLMhwuZ3JwYy50ZXN0aW5nLlJlc3BvbnNlUGFy", - "YW1zYgZwcm90bzM=")); + "KAUSGwoTYmFja2VuZF9jaGFubmVsX2lkeBgQIAEoBSJKCgtFY2hvUmVxdWVz", + "dBIPCgdtZXNzYWdlGAEgASgJEioKBXBhcmFtGAIgASgLMhsuZ3JwYy50ZXN0", + "aW5nLlJlcXVlc3RQYXJhbXMiRgoOUmVzcG9uc2VQYXJhbXMSGAoQcmVxdWVz", + "dF9kZWFkbGluZRgBIAEoAxIMCgRob3N0GAIgASgJEgwKBHBlZXIYAyABKAki", + "TAoMRWNob1Jlc3BvbnNlEg8KB21lc3NhZ2UYASABKAkSKwoFcGFyYW0YAiAB", + "KAsyHC5ncnBjLnRlc3RpbmcuUmVzcG9uc2VQYXJhbXNiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.DebugInfo), global::Grpc.Testing.DebugInfo.Parser, new[]{ "StackEntries", "Detail" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ErrorStatus), global::Grpc.Testing.ErrorStatus.Parser, new[]{ "Code", "ErrorMessage", "BinaryErrorDetails" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs", "BackendChannelIdx" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoRequest), global::Grpc.Testing.EchoRequest.Parser, new[]{ "Message", "Param" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParams), global::Grpc.Testing.ResponseParams.Parser, new[]{ "RequestDeadline", "Host", "Peer" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoResponse), global::Grpc.Testing.EchoResponse.Parser, new[]{ "Message", "Param" }, null, null, null) @@ -435,11 +435,12 @@ namespace Grpc.Testing { expectedClientIdentity_ = other.expectedClientIdentity_; skipCancelledCheck_ = other.skipCancelledCheck_; expectedTransportSecurityType_ = other.expectedTransportSecurityType_; - DebugInfo = other.debugInfo_ != null ? other.DebugInfo.Clone() : null; + debugInfo_ = other.debugInfo_ != null ? other.debugInfo_.Clone() : null; serverDie_ = other.serverDie_; binaryErrorDetails_ = other.binaryErrorDetails_; - ExpectedError = other.expectedError_ != null ? other.ExpectedError.Clone() : null; + expectedError_ = other.expectedError_ != null ? other.expectedError_.Clone() : null; serverSleepUs_ = other.serverSleepUs_; + backendChannelIdx_ = other.backendChannelIdx_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -622,6 +623,20 @@ namespace Grpc.Testing { } } + /// Field number for the "backend_channel_idx" field. + public const int BackendChannelIdxFieldNumber = 16; + private int backendChannelIdx_; + /// + /// which backend to send request to + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int BackendChannelIdx { + get { return backendChannelIdx_; } + set { + backendChannelIdx_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public override bool Equals(object other) { return Equals(other as RequestParams); @@ -650,6 +665,7 @@ namespace Grpc.Testing { if (BinaryErrorDetails != other.BinaryErrorDetails) return false; if (!object.Equals(ExpectedError, other.ExpectedError)) return false; if (ServerSleepUs != other.ServerSleepUs) return false; + if (BackendChannelIdx != other.BackendChannelIdx) return false; return Equals(_unknownFields, other._unknownFields); } @@ -671,6 +687,7 @@ namespace Grpc.Testing { if (BinaryErrorDetails.Length != 0) hash ^= BinaryErrorDetails.GetHashCode(); if (expectedError_ != null) hash ^= ExpectedError.GetHashCode(); if (ServerSleepUs != 0) hash ^= ServerSleepUs.GetHashCode(); + if (BackendChannelIdx != 0) hash ^= BackendChannelIdx.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -744,6 +761,10 @@ namespace Grpc.Testing { output.WriteRawTag(120); output.WriteInt32(ServerSleepUs); } + if (BackendChannelIdx != 0) { + output.WriteRawTag(128, 1); + output.WriteInt32(BackendChannelIdx); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -797,6 +818,9 @@ namespace Grpc.Testing { if (ServerSleepUs != 0) { size += 1 + pb::CodedOutputStream.ComputeInt32Size(ServerSleepUs); } + if (BackendChannelIdx != 0) { + size += 2 + pb::CodedOutputStream.ComputeInt32Size(BackendChannelIdx); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -859,6 +883,9 @@ namespace Grpc.Testing { if (other.ServerSleepUs != 0) { ServerSleepUs = other.ServerSleepUs; } + if (other.BackendChannelIdx != 0) { + BackendChannelIdx = other.BackendChannelIdx; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -936,6 +963,10 @@ namespace Grpc.Testing { ServerSleepUs = input.ReadInt32(); break; } + case 128: { + BackendChannelIdx = input.ReadInt32(); + break; + } } } } @@ -968,7 +999,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoRequest(EchoRequest other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1316,7 +1347,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoResponse(EchoResponse other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj index e4f36d8810d..ad7033b7829 100755 --- a/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj +++ b/src/csharp/Grpc.IntegrationTesting/Grpc.IntegrationTesting.csproj @@ -19,8 +19,8 @@ - - + + diff --git a/src/csharp/Grpc.IntegrationTesting/Messages.cs b/src/csharp/Grpc.IntegrationTesting/Messages.cs index b5c93babd20..35546f1b671 100644 --- a/src/csharp/Grpc.IntegrationTesting/Messages.cs +++ b/src/csharp/Grpc.IntegrationTesting/Messages.cs @@ -77,7 +77,6 @@ namespace Grpc.Testing { } #region Enums /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of payload that should be returned. /// public enum PayloadType { @@ -269,7 +268,6 @@ namespace Grpc.Testing { public const int TypeFieldNumber = 1; private global::Grpc.Testing.PayloadType type_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of data in body. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -585,12 +583,12 @@ namespace Grpc.Testing { public SimpleRequest(SimpleRequest other) : this() { responseType_ = other.responseType_; responseSize_ = other.responseSize_; - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; fillUsername_ = other.fillUsername_; fillOauthScope_ = other.fillOauthScope_; - ResponseCompressed = other.responseCompressed_ != null ? other.ResponseCompressed.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + responseCompressed_ = other.responseCompressed_ != null ? other.responseCompressed_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -603,7 +601,6 @@ namespace Grpc.Testing { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, server randomly chooses one from other formats. /// @@ -964,7 +961,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public SimpleResponse(SimpleResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; username_ = other.username_; oauthScope_ = other.oauthScope_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -1168,8 +1165,8 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public StreamingInputCallRequest(StreamingInputCallRequest other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1486,7 +1483,7 @@ namespace Grpc.Testing { public ResponseParameters(ResponseParameters other) : this() { size_ = other.size_; intervalUs_ = other.intervalUs_; - Compressed = other.compressed_ != null ? other.Compressed.Clone() : null; + compressed_ = other.compressed_ != null ? other.compressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1693,8 +1690,8 @@ namespace Grpc.Testing { public StreamingOutputCallRequest(StreamingOutputCallRequest other) : this() { responseType_ = other.responseType_; responseParameters_ = other.responseParameters_.Clone(); - Payload = other.payload_ != null ? other.Payload.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1707,7 +1704,6 @@ namespace Grpc.Testing { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, the payload from each response in the stream /// might be of different types. This is to simulate a mixed type of payload @@ -1927,7 +1923,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public StreamingOutputCallResponse(StreamingOutputCallResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Stats.cs b/src/csharp/Grpc.IntegrationTesting/Stats.cs index 8160646215a..af83eef7ba8 100644 --- a/src/csharp/Grpc.IntegrationTesting/Stats.cs +++ b/src/csharp/Grpc.IntegrationTesting/Stats.cs @@ -86,7 +86,7 @@ namespace Grpc.Testing { totalCpuTime_ = other.totalCpuTime_; idleCpuTime_ = other.idleCpuTime_; cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -993,13 +993,13 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStats(ClientStats other) : this() { - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; timeElapsed_ = other.timeElapsed_; timeUser_ = other.timeUser_; timeSystem_ = other.timeSystem_; requestResults_ = other.requestResults_.Clone(); cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj index d3686971245..0c12f38f25a 100755 --- a/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj +++ b/src/csharp/Grpc.Reflection.Tests/Grpc.Reflection.Tests.csproj @@ -16,8 +16,8 @@ - - + + diff --git a/src/csharp/Grpc.Reflection/Reflection.cs b/src/csharp/Grpc.Reflection/Reflection.cs index 84b2a0a8421..e319be5bff7 100644 --- a/src/csharp/Grpc.Reflection/Reflection.cs +++ b/src/csharp/Grpc.Reflection/Reflection.cs @@ -610,7 +610,7 @@ namespace Grpc.Reflection.V1Alpha { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerReflectionResponse(ServerReflectionResponse other) : this() { validHost_ = other.validHost_; - OriginalRequest = other.originalRequest_ != null ? other.OriginalRequest.Clone() : null; + originalRequest_ = other.originalRequest_ != null ? other.originalRequest_.Clone() : null; switch (other.MessageResponseCase) { case MessageResponseOneofCase.FileDescriptorResponse: FileDescriptorResponse = other.FileDescriptorResponse.Clone(); diff --git a/src/csharp/README.md b/src/csharp/README.md index 92be4bf1e54..d8aed94988f 100644 --- a/src/csharp/README.md +++ b/src/csharp/README.md @@ -32,6 +32,14 @@ HOW TO USE - To be able to generate code from Protocol Buffer (`.proto`) file definitions, add the [Grpc.Tools](https://www.nuget.org/packages/Grpc.Tools/) NuGet package that contains Protocol Buffers compiler (_protoc_) and the gRPC _protoc_ plugin. +**Xamarin.Android and Xamarin.iOS (Experimental only)** + +See [Experimentally supported platforms](experimental) for instructions. + +**Unity (Experimental only)** + +See [Experimentally supported platforms](experimental) for instructions. + BUILD FROM SOURCE ----------------- diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 67af258354f..8f38f0aa204 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -21,18 +21,12 @@ set DOTNET=dotnet mkdir ..\..\artifacts -@rem Collect the artifacts built by the previous build step if running on Jenkins +@rem Collect the artifacts built by the previous build step mkdir nativelibs -@rem Jenkins flow (deprecated) -powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs" -@rem Kokoro flow powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs" @rem Collect protoc artifacts built by the previous build step mkdir protoc_plugins -@rem Jenkins flow (deprecated) -powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins" -@rem Kokoro flow powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins" %DOTNET% restore Grpc.sln || goto :error diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh deleted file mode 100755 index 88b4d63538d..00000000000 --- a/src/csharp/build_packages_dotnetcli.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -# Copyright 2016 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -ex - -cd $(dirname $0) - -mkdir -p ../../artifacts/ - -# Collect the artifacts built by the previous build step -mkdir -p nativelibs -# Jenkins flow (deprecated) -cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true -# Kokoro flow -cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true - -# Collect protoc artifacts built by the previous build step -mkdir -p protoc_plugins -# Jenkins flow (deprecated) -cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true -# Kokoro flow -cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true - -dotnet restore Grpc.sln - -# To be able to build, we also need to put grpc_csharp_ext to its normal location -mkdir -p ../../libs/opt -cp nativelibs/csharp_ext_linux_x64/libgrpc_csharp_ext.so ../../libs/opt - -dotnet pack --configuration Release Grpc.Core --output ../../../artifacts -dotnet pack --configuration Release Grpc.Core.Testing --output ../../../artifacts -dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts -dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts -dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts - -nuget pack Grpc.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Core.NativeDebug.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.15.0-dev" -OutputDirectory ../../artifacts - -(cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat new file mode 100644 index 00000000000..9c53114b849 --- /dev/null +++ b/src/csharp/build_unitypackage.bat @@ -0,0 +1,80 @@ +@rem Copyright 2018 The gRPC Authors +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. + +@rem Current package versions +set VERSION=1.15.0-dev + +@rem Adjust the location of nuget.exe +set NUGET=C:\nuget\nuget.exe +set DOTNET=dotnet + +mkdir ..\..\artifacts + +@rem Collect the artifacts built by the previous build step +mkdir nativelibs +powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs" + +@rem Collect protoc artifacts built by the previous build step +mkdir protoc_plugins +powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins" + +%DOTNET% restore Grpc.sln || goto :error + +@rem To be able to build, we also need to put grpc_csharp_ext to its normal location +xcopy /Y /I nativelibs\csharp_ext_windows_x64\grpc_csharp_ext.dll ..\..\cmake\build\x64\Release\ + +%DOTNET% build --configuration Release Grpc.Core || goto :error +@rem build HealthCheck to get hold of Google.Protobuf.dll assembly +%DOTNET% build --configuration Release Grpc.HealthCheck || goto :error + +@rem copy Grpc assemblies to the unity package skeleton +@rem TODO(jtattermusch): Add Grpc.Auth assembly and its dependencies +copy /Y Grpc.Core\bin\Release\net45\Grpc.Core.dll unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\lib\net45\Grpc.Core.dll || goto :error +copy /Y Grpc.Core\bin\Release\net45\Grpc.Core.pdb unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\lib\net45\Grpc.Core.pdb || goto :error +copy /Y Grpc.Core\bin\Release\net45\Grpc.Core.xml unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\lib\net45\Grpc.Core.xml || goto :error + +@rem copy desktop native libraries to the unity package skeleton +copy /Y nativelibs\csharp_ext_linux_x86\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\linux\x86\libgrpc_csharp_ext.so || goto :error +copy /Y nativelibs\csharp_ext_linux_x64\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\linux\x64\libgrpc_csharp_ext.so || goto :error +copy /Y nativelibs\csharp_ext_macos_x86\libgrpc_csharp_ext.dylib unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\osx\x86\grpc_csharp_ext.bundle || goto :error +copy /Y nativelibs\csharp_ext_macos_x64\libgrpc_csharp_ext.dylib unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\osx\x64\grpc_csharp_ext.bundle || goto :error +copy /Y nativelibs\csharp_ext_windows_x86\grpc_csharp_ext.dll unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\win\x86\grpc_csharp_ext.dll || goto :error +copy /Y nativelibs\csharp_ext_windows_x64\grpc_csharp_ext.dll unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\win\x64\grpc_csharp_ext.dll || goto :error + +@rem add Android and iOS native libraries +copy /Y nativelibs\csharp_ext_linux_android_armeabi-v7a\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\android\armeabi-v7a\libgrpc_csharp_ext.so || goto :error +copy /Y nativelibs\csharp_ext_linux_android_arm64-v8a\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\android\arm64-v8a\libgrpc_csharp_ext.so || goto :error +copy /Y nativelibs\csharp_ext_linux_android_x86\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\android\x86\libgrpc_csharp_ext.so || goto :error +copy /Y nativelibs\csharp_ext_macos_ios\libgrpc_csharp_ext.a unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\ios\libgrpc_csharp_ext.a || goto :error +copy /Y nativelibs\csharp_ext_macos_ios\libgrpc.a unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\ios\libgrpc.a || goto :error + +@rem add gRPC dependencies +@rem TODO(jtattermusch): also include XMLdoc +copy /Y Grpc.Core\bin\Release\net45\System.Interactive.Async.dll unitypackage\unitypackage_skeleton\Plugins\System.Interactive.Async\lib\net45\System.Interactive.Async.dll || goto :error + +@rem add Google.Protobuf +@rem TODO(jtattermusch): also include XMLdoc +copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error + +@rem create a zipfile that will act as a Unity package +cd unitypackage\unitypackage_skeleton +zip -r ..\..\grpc_unity_package.zip Plugins +cd ..\.. +copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error + +goto :EOF + +:error +echo Failed! +exit /b %errorlevel% diff --git a/src/csharp/experimental/README.md b/src/csharp/experimental/README.md index f892a2ee95a..bd53cbcd357 100644 --- a/src/csharp/experimental/README.md +++ b/src/csharp/experimental/README.md @@ -1,16 +1,35 @@ This directory contains useful resources for getting gRPC C# to work on -not-yet-supported platforms. +platforms that are not yet fully supported. -# Unity & Xamarin -gRPC C# currently doesn't support Unity or Xamarin, but some proof-of-concept -work has been done. Some of the resources are shared in this directory to -ease community work on Unity & Xamarin support. +# Xamarin -## Crosscompiling `grpc_csharp_ext` for Android +gRPC C# now has experimental support for Xamarin. +See [HelloworldXamarin](/examples/csharp/HelloworldXamarin) for an example how to use it. -* Install [Android NDK](https://developer.android.com/ndk/index.html) -* Run `./build_native_ext_for_android.sh` to crosscompile using cmake. +What's currently supported: -## Crosscompiling `grpc_csharp_ext` for iOS +Xamarin.Android +- supported API level: Kitkat 4.4+ (= API level 19) +- supported ABIs: `armeabi-v7a` (vast majority of Android devices out there), + `arm64-v8a` (some newer Android devices), `x86` (for emulator) -TBD +Xamarin.iOS +- supported architectures: arm64 (iPhone 6+) and x86_64 (iPhone simulator) + +# Unity + +gRPC C# now has experimental support for Unity. Please try using gRPC with +Unity and provide feedback! + +How to test gRPC in a Unity project +1. Create a Unity project that targets .NET 4.x (Edit -> Project Settings -> Editor -> Scripting Runtime Version). gRPC uses APIs that are only available in .NET4.5+ so this is a requirement. +2. Download the latest development build of `grpc_unity_package.VERSION.zip` from + [daily builds](https://packages.grpc.io/) +3. Extract the `.zip` file in the `Assets` directory in your Unity project +4. Unity IDE will pick up all the bundled files and add them to project automatically. + You should be able to use gRPC and Protobuf in your scripts from now on. + +What's currently bundled in the `grpc_unity_package` +- Grpc.Core and its dependencies +- Google.Protobuf +- Precompiled native libraries for Linux, MacOS, Windows, Android and iOS. diff --git a/src/csharp/experimental/build_native_ext_for_ios.sh b/src/csharp/experimental/build_native_ext_for_ios.sh new file mode 100755 index 00000000000..69c9cdf021c --- /dev/null +++ b/src/csharp/experimental/build_native_ext_for_ios.sh @@ -0,0 +1,62 @@ +#!/bin/sh +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Helper script to crosscompile grpc_csharp_ext native extension for Android. + +set -ex + +cd "$(dirname "$0")/../../.." + +# Usage: build +function build { + SDK="$1" + ARCH="$2" + + PATH_AR="$(xcrun --sdk $SDK --find ar)" + PATH_CC="$(xcrun --sdk $SDK --find clang)" + PATH_CXX="$(xcrun --sdk $SDK --find clang++)" + + # TODO(jtattermusch): add -mios-version-min=6.0 and -Wl,ios_version_min=6.0 + CPPFLAGS="-O2 -Wframe-larger-than=16384 -arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path) -DPB_NO_PACKED_STRUCTS=1" + LDFLAGS="-arch $ARCH -isysroot $(xcrun --sdk $SDK --show-sdk-path)" + + # TODO(jtattermusch): revisit the build arguments + make -j4 static_csharp \ + VALID_CONFIG_ios_$ARCH="1" \ + CC_ios_$ARCH="$PATH_CC" \ + CXX_ios_$ARCH="$PATH_CXX" \ + LD_ios_$ARCH="$PATH_CC" \ + LDXX_ios_$ARCH="$PATH_CXX" \ + CPPFLAGS_ios_$ARCH="$CPPFLAGS" \ + LDFLAGS_ios_$ARCH="$LDFLAGS" \ + DEFINES_ios_$ARCH="NDEBUG" \ + CONFIG="ios_$ARCH" +} + +# Usage: fatten +function fatten { + LIB_NAME="$1" + + mkdir -p libs/ios + lipo -create -output libs/ios/lib$LIB_NAME.a \ + libs/ios_arm64/lib$LIB_NAME.a \ + libs/ios_x86_64/lib$LIB_NAME.a +} + +build iphoneos arm64 +build iphonesimulator x86_64 + +fatten grpc +fatten grpc_csharp_ext diff --git a/src/csharp/unitypackage/README.md b/src/csharp/unitypackage/README.md new file mode 100644 index 00000000000..aa8ffb2eca9 --- /dev/null +++ b/src/csharp/unitypackage/README.md @@ -0,0 +1,13 @@ +# Scripts for building gRPC C# package for Unity + +Scripts in this directory are of no interest for end users. They are part +of internal tooling to automate building of the gRPC C# package for Unity. + +- `unitypackage_skeleton` - preconfigured `.meta` files for the unity package + layout. The actual assemblies and native libraries will be added into + this hierarchy while building the package. + Note: The `.meta` file were created by the Unity IDE by manually adding the assemblies/native libraries + to a Unity project and configuring their target plaform/architecture in the UI (these setting get recorded in + `.meta` files). The `.meta` format is not very well documented and there seems to be no easy way to generate them + automatically. + \ No newline at end of file diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf.meta new file mode 100644 index 00000000000..c9d3ca96b71 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 4b6ebd913ac610041b63fc847ee96302 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib.meta new file mode 100644 index 00000000000..d415abf6ddb --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: e4ab5b3f8a676a84a9eb3ffcf1e8138d +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45.meta new file mode 100644 index 00000000000..b7670b6f723 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 4c50e24955a9a294b99ad097ae2e252a +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45/Google.Protobuf.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45/Google.Protobuf.dll.meta new file mode 100644 index 00000000000..d6a537838eb --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45/Google.Protobuf.dll.meta @@ -0,0 +1,30 @@ +fileFormatVersion: 2 +guid: 286edf5aec6aae64697e5e8b7ab9b60e +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45/Google.Protobuf.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45/Google.Protobuf.xml.meta new file mode 100644 index 00000000000..2b706c1910d --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Google.Protobuf/lib/net45/Google.Protobuf.xml.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 22de11620a7e57e41bc221729937524c +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core.meta new file mode 100644 index 00000000000..47737027eea --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: e28eaddc034fe664591e93e8816968aa +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib.meta new file mode 100644 index 00000000000..0e284f0be56 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 1f862d451f11a2442aeac99ad2827285 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45.meta new file mode 100644 index 00000000000..85acb0b286b --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 89bdbe2a1282d4e40ba11e96b1481d09 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.dll.meta new file mode 100644 index 00000000000..f22a7c4c3b1 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.dll.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: 63fdcd70d29d1dc49b4e61e22a9e96e1 +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.pdb.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.pdb.meta new file mode 100644 index 00000000000..d3390e96731 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.pdb.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 49a6ef383d4d43640b6c7fd8168415ae +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.xml.meta new file mode 100644 index 00000000000..2c1a532920a --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/lib/net45/Grpc.Core.xml.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: ec9cf7702008ac648af0687c05064f78 +timeCreated: 1531219386 +licenseType: Free +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes.meta new file mode 100644 index 00000000000..921669fdd34 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 643bdb9ca01174f4eb77e3e15f8324b3 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android.meta new file mode 100644 index 00000000000..489e8ddd46d --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: f7f79be62cf24a940be0211dab5d2e5e +folderAsset: yes +timeCreated: 1531220113 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a.meta new file mode 100644 index 00000000000..976b74b6e25 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 47ba39f5fad6b49c5aae011940e9eea2 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta new file mode 100644 index 00000000000..6f93617280d --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta @@ -0,0 +1,102 @@ +fileFormatVersion: 2 +guid: e1f44cc7ecd4244448817ccae6de42a3 +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 0 + Exclude Editor: 1 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXUniversal: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 1 + settings: + CPU: ARM64 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + AddToEmbeddedBinaries: false + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a.meta new file mode 100644 index 00000000000..36a3b7e6ca5 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: de2aa4d10bc38f54fb8345aca9b21195 +folderAsset: yes +timeCreated: 1531220167 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta new file mode 100644 index 00000000000..82578436b83 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta @@ -0,0 +1,102 @@ +fileFormatVersion: 2 +guid: 04fe0e4dcf310416b991e57c99e5d55f +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 0 + Exclude Editor: 1 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXUniversal: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 1 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + AddToEmbeddedBinaries: false + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86.meta new file mode 100644 index 00000000000..53032587eb0 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 800ab3aa15d1b49b19df1caf1315bd3e +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta new file mode 100644 index 00000000000..3ebdd010e88 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta @@ -0,0 +1,102 @@ +fileFormatVersion: 2 +guid: 245e3d515096b414fbcdd1fd4160161a +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 0 + Exclude Editor: 1 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXUniversal: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 1 + settings: + CPU: x86 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + AddToEmbeddedBinaries: false + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios.meta new file mode 100644 index 00000000000..5d3259e2095 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: a5f54f019e6510541ac61eeec96aa82d +folderAsset: yes +timeCreated: 1531456012 +licenseType: Pro +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios/libgrpc.a.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios/libgrpc.a.meta new file mode 100644 index 00000000000..3647568a8b2 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios/libgrpc.a.meta @@ -0,0 +1,95 @@ +fileFormatVersion: 2 +guid: dde9f75258bae4033a4f778fd1149ccd +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Editor: 1 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXUniversal: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 0 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 1 + settings: + AddToEmbeddedBinaries: false + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios/libgrpc_csharp_ext.a.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios/libgrpc_csharp_ext.a.meta new file mode 100644 index 00000000000..af8485851f8 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/ios/libgrpc_csharp_ext.a.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: 83f99c0e7db71754ba8927bbfb83c668 +timeCreated: 1531456013 +licenseType: Pro +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 1 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXIntel: 1 + Exclude OSXIntel64: 1 + Exclude OSXUniversal: 1 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 0 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + CPU: AnyCPU + DefaultValueInitialized: true + OS: AnyOS + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXIntel + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: OSXIntel64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 1 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux.meta new file mode 100644 index 00000000000..9fc0e647855 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 1ff40de7dc502c14e8893c7679e9f44a +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x64.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x64.meta new file mode 100644 index 00000000000..28d3c0fd8db --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x64.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 08ccfdbfbd8d2d6458eb44d41a7b0b0a +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x64/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x64/libgrpc_csharp_ext.so.meta new file mode 100644 index 00000000000..0d639762e8e --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x64/libgrpc_csharp_ext.so.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: c176dc12f2803e14bb37bcd0ba4a0718 +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux: 1 + Exclude Linux64: 0 + Exclude LinuxUniversal: 0 + Exclude OSXIntel: 1 + Exclude OSXIntel64: 1 + Exclude OSXUniversal: 1 + Exclude Win: 0 + Exclude Win64: 0 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86_64 + DefaultValueInitialized: true + OS: Linux + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Linux64 + second: + enabled: 1 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 1 + settings: + CPU: x86_64 + - first: + Standalone: OSXIntel + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXIntel64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x86.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x86.meta new file mode 100644 index 00000000000..b050a3b23fe --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x86.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: d62d998e3cdebd542a967743103c64b6 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x86/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x86/libgrpc_csharp_ext.so.meta new file mode 100644 index 00000000000..f597b1aa230 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/linux/x86/libgrpc_csharp_ext.so.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: 0165d78fa73d0cc4c8984f3d4f2a6c86 +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux: 0 + Exclude Linux64: 1 + Exclude LinuxUniversal: 0 + Exclude OSXIntel: 1 + Exclude OSXIntel64: 1 + Exclude OSXUniversal: 1 + Exclude Win: 0 + Exclude Win64: 0 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86 + DefaultValueInitialized: true + OS: Linux + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 1 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: LinuxUniversal + second: + enabled: 1 + settings: + CPU: x86 + - first: + Standalone: OSXIntel + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXIntel64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx.meta new file mode 100644 index 00000000000..cc9c9b60fff --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 7b1ebe888da0a174c95a712766d2558c +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x64.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x64.meta new file mode 100644 index 00000000000..d01c158c676 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x64.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: ccea531b15a71974a91329186dd65075 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x64/grpc_csharp_ext.bundle.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x64/grpc_csharp_ext.bundle.meta new file mode 100644 index 00000000000..9d192f59e5e --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x64/grpc_csharp_ext.bundle.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: da53c729add26524fbecb84a7b599289 +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXIntel: 1 + Exclude OSXIntel64: 0 + Exclude OSXUniversal: 0 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86_64 + DefaultValueInitialized: true + OS: OSX + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXIntel + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXIntel64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: x86_64 + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x86.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x86.meta new file mode 100644 index 00000000000..17713f8c945 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x86.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 2ee63d0ffcfa72a4fbd7ad04ba34cadb +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x86/grpc_csharp_ext.bundle.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x86/grpc_csharp_ext.bundle.meta new file mode 100644 index 00000000000..53822bbe0cf --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/osx/x86/grpc_csharp_ext.bundle.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: 34ec9b69f128d6d4fa3fb46bc70b547c +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux: 1 + Exclude Linux64: 1 + Exclude LinuxUniversal: 1 + Exclude OSXIntel: 0 + Exclude OSXIntel64: 1 + Exclude OSXUniversal: 0 + Exclude Win: 1 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86 + DefaultValueInitialized: true + OS: OSX + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 0 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 0 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXIntel + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXIntel64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: x86 + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win.meta new file mode 100644 index 00000000000..1491ee6f0d6 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 928a9392668000344b70b6fd126e522c +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x64.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x64.meta new file mode 100644 index 00000000000..ac2a063f989 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x64.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: 34969b8c99d7a934eaa5e84da9fcf326 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x64/grpc_csharp_ext.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x64/grpc_csharp_ext.dll.meta new file mode 100644 index 00000000000..a42f84fc8aa --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x64/grpc_csharp_ext.dll.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: cb634ddbaed75c344a6b79bc91ca996e +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux: 0 + Exclude Linux64: 0 + Exclude LinuxUniversal: 0 + Exclude OSXIntel: 0 + Exclude OSXIntel64: 0 + Exclude OSXUniversal: 0 + Exclude Win: 1 + Exclude Win64: 0 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86_64 + DefaultValueInitialized: true + OS: Windows + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Standalone: Linux + second: + enabled: 1 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 1 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXIntel + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXIntel64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Win64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + iPhone: iOS + second: + enabled: 0 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x86.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x86.meta new file mode 100644 index 00000000000..f3eb146cc89 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x86.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: ed5c6044f868f9a4f955b7db2d2df619 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x86/grpc_csharp_ext.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x86/grpc_csharp_ext.dll.meta new file mode 100644 index 00000000000..bd6291eafc3 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/win/x86/grpc_csharp_ext.dll.meta @@ -0,0 +1,117 @@ +fileFormatVersion: 2 +guid: fe9aa1338bfc3864eb50df0f9d648316 +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + '': Any + second: + enabled: 0 + settings: + Exclude Android: 1 + Exclude Editor: 0 + Exclude Linux: 0 + Exclude Linux64: 0 + Exclude LinuxUniversal: 0 + Exclude OSXIntel: 0 + Exclude OSXIntel64: 0 + Exclude OSXUniversal: 0 + Exclude Win: 0 + Exclude Win64: 1 + Exclude iOS: 1 + - first: + Android: Android + second: + enabled: 0 + settings: + CPU: ARMv7 + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + CPU: x86 + DefaultValueInitialized: true + OS: Windows + - first: + Facebook: Win + second: + enabled: 0 + settings: + CPU: AnyCPU + - first: + Facebook: Win64 + second: + enabled: 0 + settings: + CPU: None + - first: + Standalone: Linux + second: + enabled: 1 + settings: + CPU: x86 + - first: + Standalone: Linux64 + second: + enabled: 1 + settings: + CPU: x86_64 + - first: + Standalone: LinuxUniversal + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXIntel + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXIntel64 + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win + second: + enabled: 1 + settings: + CPU: AnyCPU + - first: + Standalone: Win64 + second: + enabled: 0 + settings: + CPU: None + - first: + iPhone: iOS + second: + enabled: 0 + settings: + CompileFlags: + FrameworkDependencies: + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async.meta new file mode 100644 index 00000000000..6f1b910534d --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: b2ba2199282e9c641bc2e611f1e776a0 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib.meta new file mode 100644 index 00000000000..d7ae012a397 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: bd5ddd2522dc301488ffe002106fe0ab +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45.meta new file mode 100644 index 00000000000..9b1748d3e76 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45.meta @@ -0,0 +1,10 @@ +fileFormatVersion: 2 +guid: a68443518bcd1d44ba88a871dcab0c83 +folderAsset: yes +timeCreated: 1531219385 +licenseType: Free +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45/System.Interactive.Async.dll.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45/System.Interactive.Async.dll.meta new file mode 100644 index 00000000000..bb910fe9229 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45/System.Interactive.Async.dll.meta @@ -0,0 +1,32 @@ +fileFormatVersion: 2 +guid: 4c05e46cbf00c68408f5ddc1eef9ae3b +timeCreated: 1531219386 +licenseType: Free +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + isPreloaded: 0 + isOverridable: 0 + platformData: + - first: + Any: + second: + enabled: 1 + settings: {} + - first: + Editor: Editor + second: + enabled: 0 + settings: + DefaultValueInitialized: true + - first: + Windows Store Apps: WindowsStoreApps + second: + enabled: 0 + settings: + CPU: AnyCPU + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45/System.Interactive.Async.xml.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45/System.Interactive.Async.xml.meta new file mode 100644 index 00000000000..53f91502bd6 --- /dev/null +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/System.Interactive.Async/lib/net45/System.Interactive.Async.xml.meta @@ -0,0 +1,9 @@ +fileFormatVersion: 2 +guid: 6fc38864f2d3dde46b3833c62c91a008 +timeCreated: 1531219386 +licenseType: Free +TextScriptImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 6ad9166b32e..5e9a9a45131 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index 12598e616a6..b98339941e5 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.5.0' + v = '3.6.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC diff --git a/src/objective-c/BoringSSL.podspec b/src/objective-c/BoringSSL.podspec index 363983183e9..dc566803930 100644 --- a/src/objective-c/BoringSSL.podspec +++ b/src/objective-c/BoringSSL.podspec @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.name = 'BoringSSL' - version = '10.0.5' + version = '10.0.6' s.version = version s.summary = 'BoringSSL is a fork of OpenSSL that is designed to meet Google’s needs.' # Adapted from the homepage: @@ -68,7 +68,7 @@ Pod::Spec.new do |s| s.source = { :git => 'https://github.com/google/boringssl.git', - :commit => "0c1f336fba7c8cdbe8f32a8c75a8a9f8461feff1", + :commit => "b29b21a81b32ec273f118f589f46d56ad3332420", } s.ios.deployment_target = '5.0' diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 9783b064402..084fbdeb491 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -45,6 +45,8 @@ static NSMutableDictionary *callFlags; static NSString *const kAuthorizationHeader = @"authorization"; static NSString *const kBearerPrefix = @"Bearer "; +const char *kCFStreamVarName = "grpc_cfstream"; + @interface GRPCCall () // Make them read-write. @property(atomic, strong) NSDictionary *responseHeaders; @@ -206,9 +208,12 @@ static NSString *const kBearerPrefix = @"Bearer "; } else { [_responseWriteable enqueueSuccessfulCompletion]; } -#ifndef GRPC_CFSTREAM - [GRPCConnectivityMonitor unregisterObserver:self]; -#endif + + // Connectivity monitor is not required for CFStream + char *enableCFStream = getenv(kCFStreamVarName); + if (enableCFStream == nil || enableCFStream[0] != '1') { + [GRPCConnectivityMonitor unregisterObserver:self]; + } // If the call isn't retained anywhere else, it can be deallocated now. _retainSelf = nil; @@ -220,17 +225,16 @@ static NSString *const kBearerPrefix = @"Bearer "; } - (void)cancel { - [self - maybeFinishWithError:[NSError - errorWithDomain:kGRPCErrorDomain - code:GRPCErrorCodeCancelled - userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]]; - if (!self.isWaitingForToken) { [self cancelCall]; } else { self.isWaitingForToken = NO; } + [self + maybeFinishWithError:[NSError + errorWithDomain:kGRPCErrorDomain + code:GRPCErrorCodeCancelled + userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]]; } - (void)maybeFinishWithError:(NSError *)errorOrNil { @@ -292,6 +296,7 @@ static NSString *const kBearerPrefix = @"Bearer "; // don't want to throw, because the app shouldn't crash for a behavior // that's on the hands of any server to have. Instead we finish and ask // the server to cancel. + [strongSelf cancelCall]; [strongSelf maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeResourceExhausted @@ -300,7 +305,6 @@ static NSString *const kBearerPrefix = @"Bearer "; @"Client does not have enough memory to " @"hold the server response." }]]; - [strongSelf cancelCall]; return; } [strongWriteable enqueueValue:data @@ -463,9 +467,11 @@ static NSString *const kBearerPrefix = @"Bearer "; [self sendHeaders:_requestHeaders]; [self invokeCall]; -#ifndef GRPC_CFSTREAM - [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChanged:)]; -#endif + // Connectivity monitor is not required for CFStream + char *enableCFStream = getenv(kCFStreamVarName); + if (enableCFStream == nil || enableCFStream[0] != '1') { + [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChanged:)]; + } } - (void)startWithWriteable:(id)writeable { @@ -530,13 +536,17 @@ static NSString *const kBearerPrefix = @"Bearer "; } - (void)connectivityChanged:(NSNotification *)note { - [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain + // Cancel underlying call upon this notification + __strong GRPCCall *strongSelf = self; + if (strongSelf) { + [self cancelCall]; + [self + maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeUnavailable userInfo:@{ NSLocalizedDescriptionKey : @"Connectivity lost." }]]; - // Cancel underlying call upon this notification - [self cancelCall]; + } } @end diff --git a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m index bda1c3360be..11e7231def0 100644 --- a/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m +++ b/src/objective-c/GRPCClient/private/GRPCCompletionQueue.m @@ -20,13 +20,8 @@ #import -#ifdef GRPC_CFSTREAM -const grpc_completion_queue_attributes kCompletionQueueAttr = {GRPC_CQ_CURRENT_VERSION, - GRPC_CQ_NEXT, GRPC_CQ_NON_POLLING}; -#else const grpc_completion_queue_attributes kCompletionQueueAttr = { - GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING}; -#endif + GRPC_CQ_CURRENT_VERSION, GRPC_CQ_NEXT, GRPC_CQ_DEFAULT_POLLING, NULL}; @implementation GRPCCompletionQueue diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m index 2e9f9f243b3..862909f2384 100644 --- a/src/objective-c/GRPCClient/private/GRPCHost.m +++ b/src/objective-c/GRPCClient/private/GRPCHost.m @@ -34,6 +34,8 @@ NS_ASSUME_NONNULL_BEGIN +extern const char *kCFStreamVarName; + static NSMutableDictionary *kHostCache; @implementation GRPCHost { @@ -49,9 +51,11 @@ static NSMutableDictionary *kHostCache; if (_channelCreds != nil) { grpc_channel_credentials_release(_channelCreds); } -#ifndef GRPC_CFSTREAM - [GRPCConnectivityMonitor unregisterObserver:self]; -#endif + // Connectivity monitor is not required for CFStream + char *enableCFStream = getenv(kCFStreamVarName); + if (enableCFStream == nil || enableCFStream[0] != '1') { + [GRPCConnectivityMonitor unregisterObserver:self]; + } } // Default initializer. @@ -87,9 +91,12 @@ static NSMutableDictionary *kHostCache; _compressAlgorithm = GRPC_COMPRESS_NONE; _retryEnabled = YES; } -#ifndef GRPC_CFSTREAM - [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChange:)]; -#endif + + // Connectivity monitor is not required for CFStream + char *enableCFStream = getenv(kCFStreamVarName); + if (enableCFStream == nil || enableCFStream[0] != '1') { + [GRPCConnectivityMonitor registerObserver:self selector:@selector(connectivityChange:)]; + } } return self; } diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index f28e494868e..7781d27ca4f 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -187,6 +187,7 @@ grpc_slice _details; size_t _detailsCapacity; grpc_metadata_array _trailers; + const char *_errorString; } - (instancetype)init { @@ -200,6 +201,7 @@ _op.data.recv_status_on_client.status_details = &_details; grpc_metadata_array_init(&_trailers); _op.data.recv_status_on_client.trailing_metadata = &_trailers; + _op.data.recv_status_on_client.error_string = &_errorString; if (handler) { // Prevent reference cycle with _handler __weak typeof(self) weakSelf = self; @@ -207,8 +209,9 @@ __strong typeof(self) strongSelf = weakSelf; if (strongSelf) { char *details = grpc_slice_to_c_string(strongSelf->_details); - NSError *error = - [NSError grpc_errorFromStatusCode:strongSelf->_statusCode details:details]; + NSError *error = [NSError grpc_errorFromStatusCode:strongSelf->_statusCode + details:details + errorString:strongSelf->_errorString]; NSDictionary *trailers = [NSDictionary grpc_dictionaryFromMetadataArray:strongSelf->_trailers]; handler(error, trailers); @@ -223,6 +226,7 @@ - (void)dealloc { grpc_metadata_array_destroy(&_trailers); grpc_slice_unref(_details); + gpr_free((void *)_errorString); } @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.h b/src/objective-c/GRPCClient/private/NSError+GRPC.h index e96b7297c28..a63e76ee4d0 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.h +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.h @@ -24,5 +24,7 @@ * Returns nil if the status code is OK. Otherwise, a NSError whose code is one of |GRPCErrorCode| * and whose domain is |kGRPCErrorDomain|. */ -+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details; ++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode + details:(char *)details + errorString:(const char *)errorString; @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.m b/src/objective-c/GRPCClient/private/NSError+GRPC.m index c2e65e4d8a2..199b2ebb6c3 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.m +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.m @@ -23,13 +23,19 @@ NSString *const kGRPCErrorDomain = @"io.grpc"; @implementation NSError (GRPC) -+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details { ++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode + details:(char *)details + errorString:(const char *)errorString { if (statusCode == GRPC_STATUS_OK) { return nil; } NSString *message = [NSString stringWithCString:details encoding:NSUTF8StringEncoding]; + NSString *debugMessage = [NSString stringWithCString:errorString encoding:NSUTF8StringEncoding]; return [NSError errorWithDomain:kGRPCErrorDomain code:statusCode - userInfo:@{NSLocalizedDescriptionKey : message}]; + userInfo:@{ + NSLocalizedDescriptionKey : message, + NSDebugDescriptionErrorKey : debugMessage + }]; } @end diff --git a/src/objective-c/README-CFSTREAM.md b/src/objective-c/README-CFSTREAM.md index 0b5215a7b3b..0cb25ab237b 100644 --- a/src/objective-c/README-CFSTREAM.md +++ b/src/objective-c/README-CFSTREAM.md @@ -13,17 +13,17 @@ interface that gRPC uses when it is ready for production. ## Usage If you use gRPC following the instructions in [README.md](https://github.com/grpc/grpc/blob/master/src/objective-c/README.md): -- Simply replace the -dependency on `gRPC-ProtoRPC` with `gRPC-ProtoRPC/CFStream`. The build system will take care of -everything else and switch networking to CFStream. +- Replace the +dependency on `gRPC-ProtoRPC` with `gRPC-ProtoRPC/CFStream`. +- Enable CFStream with environment variable `grpc_cfstream=1`. This can be done either in Xcode + console or by your code with `setenv()` before gRPC is initialized. If your project directly depends on podspecs other than `gRPC-ProtoRPC` (e.g. `gRPC` or `gRPC-Core`): -- Make your projects depend on subspecs corresponding to CFStream in each gRPC podspec. For - `gRPC-Core`, you will need to make sure that the completion queue you create is of type - `GRPC_CQ_NON_POLLING`. This is expected to be fixed soon so that you do not have to modify the - completion queue type. +- Make your projects depend on subspecs corresponding to CFStream in each gRPC podspec. +- Enable CFStream with environment variable `grpc_cfstream=1`. This can be done either in Xcode + console or by your code with `setenv()` before gRPC is initialized. ## Notes diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 2a169800a09..a0de8ba8990 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -591,4 +591,39 @@ static GRPCProtoMethod *kFullDuplexCallMethod; [self testTimeoutBackoffWithTimeout:0.3 Backoff:0.7]; } +- (void)testErrorDebugInformation { + __weak XCTestExpectation *expectation = [self expectationWithDescription:@"RPC unauthorized."]; + + RMTSimpleRequest *request = [RMTSimpleRequest message]; + request.fillUsername = YES; + request.fillOauthScope = YES; + GRXWriter *requestsWriter = [GRXWriter writerWithValue:[request data]]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kRemoteSSLHost + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter]; + + call.oauth2AccessToken = @"bogusToken"; + + id responsesWriteable = + [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTFail(@"Received unexpected response: %@", value); + } + completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Finished without error!"); + NSDictionary *userInfo = errorOrNil.userInfo; + NSString *debugInformation = userInfo[NSDebugDescriptionErrorKey]; + XCTAssertNotNil(debugInformation); + XCTAssertNotEqual([debugInformation length], 0); + NSString *challengeHeader = call.oauth2ChallengeHeader; + XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@", + call.responseHeaders); + [expectation fulfill]; + }]; + + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m index 1e1da2dd664..5750dccd894 100644 --- a/src/objective-c/tests/InteropTests.m +++ b/src/objective-c/tests/InteropTests.m @@ -36,6 +36,8 @@ #define TEST_TIMEOUT 32 +extern const char *kCFStreamVarName; + // Convenience constructors for the generated proto messages: @interface RMTStreamingOutputCallRequest (Constructors) @@ -97,6 +99,9 @@ BOOL isRemoteInteropTest(NSString *host) { [Cronet start]; [GRPCCall useCronetWithEngine:[Cronet getGlobalEngine]]; #endif +#ifdef GRPC_CFSTREAM + setenv(kCFStreamVarName, "1", 1); +#endif } - (void)setUp { diff --git a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj index 8ff46335826..ea1066219d4 100644 --- a/src/objective-c/tests/Tests.xcodeproj/project.pbxproj +++ b/src/objective-c/tests/Tests.xcodeproj/project.pbxproj @@ -1982,6 +1982,16 @@ CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "COCOAPODS=1", + "$(inherited)", + "GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1", + "$(inherited)", + "PB_FIELD_32BIT=1", + "PB_NO_PACKED_STRUCTS=1", + "GRPC_CFSTREAM=1", + ); INFOPLIST_FILE = Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 11.2; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; @@ -2100,6 +2110,16 @@ CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "COCOAPODS=1", + "$(inherited)", + "GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1", + "$(inherited)", + "PB_FIELD_32BIT=1", + "PB_NO_PACKED_STRUCTS=1", + "GRPC_CFSTREAM=1", + ); INFOPLIST_FILE = Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 11.2; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; @@ -2218,6 +2238,16 @@ CODE_SIGN_IDENTITY = "iPhone Developer"; CODE_SIGN_STYLE = Automatic; GCC_C_LANGUAGE_STANDARD = gnu11; + GCC_PREPROCESSOR_DEFINITIONS = ( + "$(inherited)", + "COCOAPODS=1", + "$(inherited)", + "GPB_USE_PROTOBUF_FRAMEWORK_IMPORTS=1", + "$(inherited)", + "PB_FIELD_32BIT=1", + "PB_NO_PACKED_STRUCTS=1", + "GRPC_CFSTREAM=1", + ); INFOPLIST_FILE = Info.plist; IPHONEOS_DEPLOYMENT_TARGET = 11.2; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks"; diff --git a/src/objective-c/tests/build_one_example.sh b/src/objective-c/tests/build_one_example.sh index 1eace541e6c..084147f1d46 100755 --- a/src/objective-c/tests/build_one_example.sh +++ b/src/objective-c/tests/build_one_example.sh @@ -43,7 +43,7 @@ xcodebuild \ -workspace *.xcworkspace \ -scheme $SCHEME \ -destination generic/platform=iOS \ - -derivedDataPath Build \ + -derivedDataPath Build/Build \ CODE_SIGN_IDENTITY="" \ CODE_SIGNING_REQUIRED=NO \ | egrep -v "$XCODEBUILD_FILTER" \ diff --git a/src/php/composer.json b/src/php/composer.json index b6716b8b4c3..91913d73d29 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -8,7 +8,7 @@ "google/protobuf": "^v3.3.0" }, "require-dev": { - "google/auth": "v0.9" + "google/auth": "^v1.3.0" }, "autoload": { "psr-4": { diff --git a/src/php/ext/grpc/server.c b/src/php/ext/grpc/server.c index cb7b188b0e4..8c7eaee203f 100644 --- a/src/php/ext/grpc/server.c +++ b/src/php/ext/grpc/server.c @@ -75,7 +75,10 @@ PHP_METHOD(Server, __construct) { if (args_array == NULL) { server->wrapped = grpc_server_create(NULL, NULL); } else { - php_grpc_read_args_array(args_array, &args TSRMLS_CC); + if (php_grpc_read_args_array(args_array, &args TSRMLS_CC) == FAILURE) { + efree(args.args); + return; + } server->wrapped = grpc_server_create(&args, NULL); efree(args.args); } diff --git a/src/php/lib/Grpc/BaseStub.php b/src/php/lib/Grpc/BaseStub.php index ecb419ac8f8..fe81e377610 100644 --- a/src/php/lib/Grpc/BaseStub.php +++ b/src/php/lib/Grpc/BaseStub.php @@ -83,10 +83,11 @@ class BaseStub } private static function updateOpts($opts) { - $package_config = json_decode( - file_get_contents(dirname(__FILE__).'/../../composer.json'), - true - ); + if (!file_exists($composerFile = __DIR__.'/../../composer.json')) { + // for grpc/grpc-php subpackage + $composerFile = __DIR__.'/../composer.json'; + } + $package_config = json_decode(file_get_contents($composerFile), true); if (!empty($opts['grpc.primary_user_agent'])) { $opts['grpc.primary_user_agent'] .= ' '; } else { diff --git a/src/php/tests/unit_tests/PersistentChannelTests/PersistentChannelTest.php b/src/php/tests/unit_tests/PersistentChannelTests/PersistentChannelTest.php index 2bb5c4bb852..7515a012334 100644 --- a/src/php/tests/unit_tests/PersistentChannelTests/PersistentChannelTest.php +++ b/src/php/tests/unit_tests/PersistentChannelTests/PersistentChannelTest.php @@ -174,12 +174,12 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelSharedChannelClose() { // same underlying channel - $this->channel1 = new Grpc\Channel('localhost:10010', [ + $this->channel1 = new Grpc\Channel('localhost:10001', [ "grpc_target_persist_bound" => 2, ]); - $this->channel2 = new Grpc\Channel('localhost:10010', []); + $this->channel2 = new Grpc\Channel('localhost:10001', []); $this->server = new Grpc\Server([]); - $this->port = $this->server->addHttp2Port('localhost:10010'); + $this->port = $this->server->addHttp2Port('localhost:10001'); // channel2 can still be use $state = $this->channel2->getConnectivityState(); @@ -216,7 +216,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelTargetDefaultUpperBound() { - $this->channel1 = new Grpc\Channel('localhost:10011', []); + $this->channel1 = new Grpc\Channel('localhost:10002', []); $channel1_info = $this->channel1->getChannelInfo(); $this->assertEquals($channel1_info['target_upper_bound'], 1); $this->assertEquals($channel1_info['target_current_size'], 1); @@ -224,7 +224,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelTargetUpperBoundZero() { - $this->channel1 = new Grpc\Channel('localhost:10011', [ + $this->channel1 = new Grpc\Channel('localhost:10002', [ "grpc_target_persist_bound" => 0, ]); // channel1 will not be persisted. @@ -237,7 +237,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelTargetUpperBoundNotZero() { - $this->channel1 = new Grpc\Channel('localhost:10011', [ + $this->channel1 = new Grpc\Channel('localhost:10003', [ "grpc_target_persist_bound" => 3, ]); $channel1_info = $this->channel1->getChannelInfo(); @@ -245,7 +245,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase $this->assertEquals($channel1_info['target_current_size'], 1); // The upper bound should not be changed - $this->channel2 = new Grpc\Channel('localhost:10011', []); + $this->channel2 = new Grpc\Channel('localhost:10003', []); $channel2_info = $this->channel2->getChannelInfo(); $this->assertEquals($channel2_info['target_upper_bound'], 3); $this->assertEquals($channel2_info['target_current_size'], 1); @@ -253,14 +253,14 @@ class PersistentListTest extends PHPUnit_Framework_TestCase // The upper bound should not be changed $channel_credentials = Grpc\ChannelCredentials::createSsl(null, null, null); - $this->channel3 = new Grpc\Channel('localhost:10011', + $this->channel3 = new Grpc\Channel('localhost:10003', ['credentials' => $channel_credentials]); $channel3_info = $this->channel3->getChannelInfo(); $this->assertEquals($channel3_info['target_upper_bound'], 3); $this->assertEquals($channel3_info['target_current_size'], 2); // The upper bound should not be changed - $this->channel4 = new Grpc\Channel('localhost:10011', [ + $this->channel4 = new Grpc\Channel('localhost:10003', [ "grpc_target_persist_bound" => 5, ]); $channel4_info = $this->channel4->getChannelInfo(); @@ -393,14 +393,14 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelTwoUpperBoundOutBound2() { - $this->channel1 = new Grpc\Channel('localhost:10011', [ + $this->channel1 = new Grpc\Channel('localhost:10012', [ "grpc_target_persist_bound" => 2, ]); $channel1_info = $this->channel1->getChannelInfo(); $channel_credentials = Grpc\ChannelCredentials::createSsl(null, null, null); - $this->channel2 = new Grpc\Channel('localhost:10011', + $this->channel2 = new Grpc\Channel('localhost:10012', ['credentials' => $channel_credentials]); $channel2_info = $this->channel2->getChannelInfo(); @@ -409,7 +409,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase $channel_credentials = Grpc\ChannelCredentials::createSsl("a", null, null); - $this->channel3 = new Grpc\Channel('localhost:10011', + $this->channel3 = new Grpc\Channel('localhost:10012', ['credentials' => $channel_credentials]); $channel3_info = $this->channel3->getChannelInfo(); @@ -422,14 +422,14 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelTwoUpperBoundOutBound3() { - $this->channel1 = new Grpc\Channel('localhost:10011', [ + $this->channel1 = new Grpc\Channel('localhost:10013', [ "grpc_target_persist_bound" => 2, ]); $channel1_info = $this->channel1->getChannelInfo(); $channel_credentials = Grpc\ChannelCredentials::createSsl(null, null, null); - $this->channel2 = new Grpc\Channel('localhost:10011', + $this->channel2 = new Grpc\Channel('localhost:10013', ['credentials' => $channel_credentials]); $this->channel2->getConnectivityState(true); $this->waitUntilNotIdle($this->channel2); @@ -442,7 +442,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase $channel_credentials = Grpc\ChannelCredentials::createSsl("a", null, null); - $this->channel3 = new Grpc\Channel('localhost:10011', + $this->channel3 = new Grpc\Channel('localhost:10013', ['credentials' => $channel_credentials]); $channel3_info = $this->channel3->getChannelInfo(); @@ -456,7 +456,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase public function testPersistentChannelTwoUpperBoundOutBound4() { - $this->channel1 = new Grpc\Channel('localhost:10011', [ + $this->channel1 = new Grpc\Channel('localhost:10014', [ "grpc_target_persist_bound" => 2, ]); $this->channel1->getConnectivityState(true); @@ -466,7 +466,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase $channel_credentials = Grpc\ChannelCredentials::createSsl(null, null, null); - $this->channel2 = new Grpc\Channel('localhost:10011', + $this->channel2 = new Grpc\Channel('localhost:10014', ['credentials' => $channel_credentials]); $this->channel2->getConnectivityState(true); $this->waitUntilNotIdle($this->channel2); @@ -475,7 +475,7 @@ class PersistentListTest extends PHPUnit_Framework_TestCase $channel_credentials = Grpc\ChannelCredentials::createSsl("a", null, null); - $this->channel3 = new Grpc\Channel('localhost:10011', + $this->channel3 = new Grpc\Channel('localhost:10014', ['credentials' => $channel_credentials]); $channel3_info = $this->channel3->getChannelInfo(); diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel new file mode 100644 index 00000000000..3f214bf3b05 --- /dev/null +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -0,0 +1,82 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "grpcio", + srcs = ["__init__.py"], + deps = [ + ":utilities", + ":auth", + ":plugin_wrapping", + ":channel", + ":interceptor", + ":server", + "//src/python/grpcio/grpc/_cython:cygrpc", + "//src/python/grpcio/grpc/beta", + "//src/python/grpcio/grpc/experimental", + "//src/python/grpcio/grpc/framework", + requirement('enum34'), + requirement('six'), + ], + data = [ + "//:grpc", + ], + main = "__init__.py", + imports = ["../",], +) + +py_library( + name = "auth", + srcs = ["_auth.py"], +) + +py_library( + name = "channel", + srcs = ["_channel.py"], + deps = [ + ":common", + ":grpcio_metadata", + ], +) + +py_library( + name = "common", + srcs = ["_common.py"], +) + +py_library( + name = "grpcio_metadata", + srcs = ["_grpcio_metadata.py"], +) + +py_library( + name = "interceptor", + srcs = ["_interceptor.py"], +) + +py_library( + name = "plugin_wrapping", + srcs = ["_plugin_wrapping.py"], + deps = [ + ":common", + ] +) + +py_library( + name = "server", + srcs = ["_server.py"], + deps = [ + ":common", + ":interceptor", + ], +) + +py_library( + name = "utilities", + srcs = ["_utilities.py"], + deps = [ + ":common", + ], +) + diff --git a/src/python/grpcio/grpc/_cython/BUILD.bazel b/src/python/grpcio/grpc/_cython/BUILD.bazel new file mode 100644 index 00000000000..7124e83dee7 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/BUILD.bazel @@ -0,0 +1,46 @@ +package(default_visibility = ["//visibility:public"]) + +load("//bazel:cython_library.bzl", "pyx_library") + +pyx_library( + name = "cygrpc", + srcs = [ + "__init__.py", + "cygrpc.pxd", + "cygrpc.pyx", + "_cygrpc/grpc_string.pyx.pxi", + "_cygrpc/arguments.pyx.pxi", + "_cygrpc/call.pyx.pxi", + "_cygrpc/channel.pyx.pxi", + "_cygrpc/credentials.pyx.pxi", + "_cygrpc/completion_queue.pyx.pxi", + "_cygrpc/event.pyx.pxi", + "_cygrpc/metadata.pyx.pxi", + "_cygrpc/operation.pyx.pxi", + "_cygrpc/records.pyx.pxi", + "_cygrpc/security.pyx.pxi", + "_cygrpc/server.pyx.pxi", + "_cygrpc/tag.pyx.pxi", + "_cygrpc/time.pyx.pxi", + "_cygrpc/grpc_gevent.pyx.pxi", + "_cygrpc/grpc.pxi", + "_cygrpc/arguments.pxd.pxi", + "_cygrpc/call.pxd.pxi", + "_cygrpc/channel.pxd.pxi", + "_cygrpc/credentials.pxd.pxi", + "_cygrpc/completion_queue.pxd.pxi", + "_cygrpc/event.pxd.pxi", + "_cygrpc/metadata.pxd.pxi", + "_cygrpc/operation.pxd.pxi", + "_cygrpc/records.pxd.pxi", + "_cygrpc/security.pxd.pxi", + "_cygrpc/server.pxd.pxi", + "_cygrpc/tag.pxd.pxi", + "_cygrpc/time.pxd.pxi", + "_cygrpc/grpc_gevent.pxd.pxi", + ], + deps = [ + "//:grpc", + ], +) + diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/_hooks.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/_hooks.pxd.pxi new file mode 100644 index 00000000000..3eb10f52757 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/_hooks.pxd.pxi @@ -0,0 +1,16 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef object _custom_op_on_c_call(int op, grpc_call *call) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/_hooks.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/_hooks.pyx.pxi new file mode 100644 index 00000000000..38cf629dc24 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/_cygrpc/_hooks.pyx.pxi @@ -0,0 +1,17 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +cdef object _custom_op_on_c_call(int op, grpc_call *call): + raise NotImplementedError("No custom hooks are implemented") diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi index 853bf6f8e04..6cb1bc0c051 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pxd.pxi @@ -1,4 +1,4 @@ -# Copyright 2018 gRPC authors. +# Copyright 2018 The gRPC Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,12 @@ cdef void _destroy_pointer(void* pointer) cdef int _compare_pointer(void* first_pointer, void* second_pointer) +cdef tuple _wrap_grpc_arg(grpc_arg arg) + + +cdef grpc_arg _unwrap_grpc_arg(tuple wrapped_arg) + + cdef class _ArgumentProcessor: cdef grpc_arg c_argument diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi index aecd3d7b116..2239e26b329 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/arguments.pyx.pxi @@ -34,6 +34,22 @@ cdef int _compare_pointer(void* first_pointer, void* second_pointer): return 0 +cdef class _GrpcArgWrapper: + + cdef grpc_arg arg + + +cdef tuple _wrap_grpc_arg(grpc_arg arg): + wrapped = _GrpcArgWrapper() + wrapped.arg = arg + return ("grpc.python._cygrpc._GrpcArgWrapper", wrapped) + + +cdef grpc_arg _unwrap_grpc_arg(tuple wrapped_arg): + cdef _GrpcArgWrapper wrapped = wrapped_arg[1] + return wrapped.arg + + cdef class _ArgumentProcessor: cdef void c(self, argument, grpc_arg_pointer_vtable *vtable, references): @@ -51,6 +67,8 @@ cdef class _ArgumentProcessor: if encoded_value is not value: references.append(encoded_value) self.c_argument.value.string = encoded_value + elif isinstance(value, _GrpcArgWrapper): + self.c_argument = (<_GrpcArgWrapper>value).arg elif hasattr(value, '__int__'): # Pointer objects must override __int__() to return # the underlying C address (Python ints are word size). The diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi index 2e02111ddd9..a0de862d947 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/call.pyx.pxi @@ -94,3 +94,5 @@ cdef class Call: def is_valid(self): return self.c_call != NULL + def _custom_op_on_c_call(self, int op): + return _custom_op_on_c_call(op, self.c_call) diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 8c37a3cf852..aa187e88a62 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -300,7 +300,7 @@ cdef class SegregatedCall: def next_event(self): def on_success(tag): _process_segregated_call_tag( - self._channel_state, self._call_state, self._c_completion_queue, tag) + self._channel_state, self._call_state, self._c_completion_queue, tag) return _next_call_event( self._channel_state, self._c_completion_queue, on_success) @@ -309,13 +309,18 @@ cdef SegregatedCall _segregated_call( _ChannelState state, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_user_tags): cdef _CallState call_state = _CallState() - cdef grpc_completion_queue *c_completion_queue = ( - grpc_completion_queue_create_for_next(NULL)) cdef SegregatedCall segregated_call + cdef grpc_completion_queue *c_completion_queue def on_success(started_tags): state.segregated_call_states.add(call_state) + with state.condition: + if state.open: + c_completion_queue = (grpc_completion_queue_create_for_next(NULL)) + else: + raise ValueError('Cannot invoke RPC on closed channel!') + try: _call( state, call_state, c_completion_queue, on_success, flags, method, host, @@ -443,8 +448,11 @@ cdef class Channel: def check_connectivity_state(self, bint try_to_connect): with self._state.condition: - return grpc_channel_check_connectivity_state( - self._state.c_channel, try_to_connect) + if self._state.open: + return grpc_channel_check_connectivity_state( + self._state.c_channel, try_to_connect) + else: + raise ValueError('Cannot invoke RPC on closed channel!') def watch_connectivity_state( self, grpc_connectivity_state last_observed_state, object deadline): diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi index d2c0389ca6a..0a25218e19e 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/credentials.pyx.pxi @@ -48,7 +48,7 @@ cdef int _get_metadata( cdef size_t metadata_count cdef grpc_metadata *c_metadata def callback(metadata, grpc_status_code status, bytes error_details): - if status is StatusCode.ok: + if status == StatusCode.ok: _store_c_metadata(metadata, &c_metadata, &metadata_count) cb(user_data, c_metadata, metadata_count, status, NULL) _release_c_metadata(c_metadata, metadata_count) diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pxd b/src/python/grpcio/grpc/_cython/cygrpc.pxd index e33c01c28f5..0cc26bc0d05 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pxd +++ b/src/python/grpcio/grpc/_cython/cygrpc.pxd @@ -28,5 +28,6 @@ include "_cygrpc/security.pxd.pxi" include "_cygrpc/server.pxd.pxi" include "_cygrpc/tag.pxd.pxi" include "_cygrpc/time.pxd.pxi" +include "_cygrpc/_hooks.pxd.pxi" include "_cygrpc/grpc_gevent.pxd.pxi" diff --git a/src/python/grpcio/grpc/_cython/cygrpc.pyx b/src/python/grpcio/grpc/_cython/cygrpc.pyx index fb16fb16bfb..3cac406687c 100644 --- a/src/python/grpcio/grpc/_cython/cygrpc.pyx +++ b/src/python/grpcio/grpc/_cython/cygrpc.pyx @@ -35,6 +35,7 @@ include "_cygrpc/security.pyx.pxi" include "_cygrpc/server.pyx.pxi" include "_cygrpc/tag.pyx.pxi" include "_cygrpc/time.pyx.pxi" +include "_cygrpc/_hooks.pyx.pxi" include "_cygrpc/grpc_gevent.pyx.pxi" diff --git a/src/python/grpcio/grpc/beta/BUILD.bazel b/src/python/grpcio/grpc/beta/BUILD.bazel new file mode 100644 index 00000000000..731be5cb25b --- /dev/null +++ b/src/python/grpcio/grpc/beta/BUILD.bazel @@ -0,0 +1,58 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "beta", + srcs = ["__init__.py",], + deps = [ + ":client_adaptations", + ":metadata", + ":server_adaptations", + ":implementations", + ":interfaces", + ":utilities", + ], +) + +py_library( + name = "client_adaptations", + srcs = ["_client_adaptations.py"], + imports=["../../",] +) + +py_library( + name = "metadata", + srcs = ["_metadata.py"], +) + +py_library( + name = "server_adaptations", + srcs = ["_server_adaptations.py"], + imports=["../../",], +) + +py_library( + name = "implementations", + srcs = ["implementations.py"], + imports=["../../",], +) + +py_library( + name = "interfaces", + srcs = ["interfaces.py"], + deps = [ + requirement("six"), + ], + imports=["../../",], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + ":implementations", + ":interfaces", + "//src/python/grpcio/grpc/framework/foundation", + ], +) + diff --git a/src/python/grpcio/grpc/experimental/BUILD.bazel b/src/python/grpcio/grpc/experimental/BUILD.bazel new file mode 100644 index 00000000000..6598d02747b --- /dev/null +++ b/src/python/grpcio/grpc/experimental/BUILD.bazel @@ -0,0 +1,27 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "experimental", + srcs = ["__init__.py",], + deps = [ + ":gevent", + ":session_cache", + ], +) + +py_library( + name = "gevent", + srcs = ["gevent.py"], + deps = [ + "//src/python/grpcio/grpc/_cython:cygrpc", + ], +) + +py_library( + name = "session_cache", + srcs = ["session_cache.py"], + deps = [ + "//src/python/grpcio/grpc/_cython:cygrpc", + ], +) diff --git a/src/python/grpcio/grpc/framework/BUILD.bazel b/src/python/grpcio/grpc/framework/BUILD.bazel new file mode 100644 index 00000000000..55b4f4d2dfe --- /dev/null +++ b/src/python/grpcio/grpc/framework/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "framework", + srcs = ["__init__.py",], + deps = [ + "//src/python/grpcio/grpc/framework/common", + "//src/python/grpcio/grpc/framework/foundation", + "//src/python/grpcio/grpc/framework/interfaces", + ], +) diff --git a/src/python/grpcio/grpc/framework/common/BUILD.bazel b/src/python/grpcio/grpc/framework/common/BUILD.bazel new file mode 100644 index 00000000000..9d9ef682c90 --- /dev/null +++ b/src/python/grpcio/grpc/framework/common/BUILD.bazel @@ -0,0 +1,27 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "common", + srcs = ["__init__.py",], + deps = [ + ":cardinality", + ":style", + ], +) + +py_library( + name = "cardinality", + srcs = ["cardinality.py"], + deps = [ + requirement("enum34"), + ], +) + +py_library( + name = "style", + srcs = ["style.py"], + deps = [ + requirement("enum34"), + ], +) diff --git a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel new file mode 100644 index 00000000000..1287fdd44ed --- /dev/null +++ b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel @@ -0,0 +1,61 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "foundation", + srcs = ["__init__.py",], + deps = [ + ":abandonment", + ":callable_util", + ":future", + ":logging_pool", + ":stream_util", + ":stream", + ], +) + +py_library( + name = "abandonment", + srcs = ["abandonment.py"], +) + +py_library( + name = "callable_util", + srcs = ["callable_util.py"], + deps = [ + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "future", + srcs = ["future.py"], + deps = [ + requirement("six"), + ], +) + +py_library( + name = "logging_pool", + srcs = ["logging_pool.py"], + deps = [ + requirement("futures"), + ], +) + +py_library( + name = "stream_util", + srcs = ["stream_util.py"], + deps = [ + ":stream", + ], +) + +py_library( + name = "stream", + srcs = ["stream.py"], + deps = [ + requirement("six"), + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel new file mode 100644 index 00000000000..b81e196cc3c --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "interfaces", + srcs = ["__init__.py",], + deps = [ + "//src/python/grpcio/grpc/framework/interfaces/base", + "//src/python/grpcio/grpc/framework/interfaces/face", + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel new file mode 100644 index 00000000000..408a66a6310 --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel @@ -0,0 +1,29 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "base_lib", + srcs = ["__init__.py",], + deps = [ + ":base", + ":utilities", + ], +) + +py_library( + name = "base", + srcs = ["base.py"], + deps = [ + "//src/python/grpcio/grpc/framework/foundation:abandonment", + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + requirement("enum34"), + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel new file mode 100644 index 00000000000..e683e7cc426 --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel @@ -0,0 +1,32 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "face", + srcs = ["__init__.py",], + deps = [ + ":face_lib", + ":utilities", + ], +) + +py_library( + name = "face_lib", + srcs = ["face.py"], + deps = [ + "//src/python/grpcio/grpc/framework/foundation", + "//src/python/grpcio/grpc/framework/common", + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + "//src/python/grpcio/grpc/framework/common", + "//src/python/grpcio/grpc/framework/foundation:stream", + ":face_lib", + ], +) diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 49185cc6487..6e6d756eec7 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -254,6 +254,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', @@ -354,11 +356,14 @@ CORE_SOURCE_FILES = [ 'src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc', 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc', + 'src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc', 'src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc', 'src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc', - 'src/cpp/ext/filters/census/grpc_context.cc', + 'src/core/ext/filters/census/grpc_context.cc', 'src/core/ext/filters/max_age/max_age_filter.cc', 'src/core/ext/filters/message_size/message_size_filter.cc', 'src/core/ext/filters/http/client_authority_filter.cc', diff --git a/src/python/grpcio_health_checking/grpc_health/v1/health.py b/src/python/grpcio_health_checking/grpc_health/v1/health.py index c8498104b12..05836594281 100644 --- a/src/python/grpcio_health_checking/grpc_health/v1/health.py +++ b/src/python/grpcio_health_checking/grpc_health/v1/health.py @@ -17,11 +17,13 @@ import threading import grpc -from grpc_health.v1 import health_pb2 -from grpc_health.v1 import health_pb2_grpc +from grpc_health.v1 import health_pb2 as _health_pb2 +from grpc_health.v1 import health_pb2_grpc as _health_pb2_grpc +SERVICE_NAME = _health_pb2.DESCRIPTOR.services_by_name['Health'].full_name -class HealthServicer(health_pb2_grpc.HealthServicer): + +class HealthServicer(_health_pb2_grpc.HealthServicer): """Servicer handling RPCs for service statuses.""" def __init__(self): @@ -33,9 +35,9 @@ class HealthServicer(health_pb2_grpc.HealthServicer): status = self._server_status.get(request.service) if status is None: context.set_code(grpc.StatusCode.NOT_FOUND) - return health_pb2.HealthCheckResponse() + return _health_pb2.HealthCheckResponse() else: - return health_pb2.HealthCheckResponse(status=status) + return _health_pb2.HealthCheckResponse(status=status) def set(self, service, status): """Sets the status of a service. diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 35c09827ba7..db2edae2cea 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -57,7 +57,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py index 0c564f10e5b..6df1a364269 100644 --- a/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py +++ b/src/python/grpcio_reflection/grpc_reflection/v1alpha/reflection.py @@ -17,15 +17,17 @@ import grpc from google.protobuf import descriptor_pb2 from google.protobuf import descriptor_pool -from grpc_reflection.v1alpha import reflection_pb2 -from grpc_reflection.v1alpha import reflection_pb2_grpc +from grpc_reflection.v1alpha import reflection_pb2 as _reflection_pb2 +from grpc_reflection.v1alpha import reflection_pb2_grpc as _reflection_pb2_grpc _POOL = descriptor_pool.Default() +SERVICE_NAME = _reflection_pb2.DESCRIPTOR.services_by_name[ + 'ServerReflection'].full_name def _not_found_error(): - return reflection_pb2.ServerReflectionResponse( - error_response=reflection_pb2.ErrorResponse( + return _reflection_pb2.ServerReflectionResponse( + error_response=_reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.NOT_FOUND.value[0], error_message=grpc.StatusCode.NOT_FOUND.value[1].encode(), )) @@ -35,12 +37,12 @@ def _file_descriptor_response(descriptor): proto = descriptor_pb2.FileDescriptorProto() descriptor.CopyToProto(proto) serialized_proto = proto.SerializeToString() - return reflection_pb2.ServerReflectionResponse( - file_descriptor_response=reflection_pb2.FileDescriptorResponse( + return _reflection_pb2.ServerReflectionResponse( + file_descriptor_response=_reflection_pb2.FileDescriptorResponse( file_descriptor_proto=(serialized_proto,)),) -class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): +class ReflectionServicer(_reflection_pb2_grpc.ServerReflectionServicer): """Servicer handling RPCs for service statuses.""" def __init__(self, service_names, pool=None): @@ -94,17 +96,17 @@ class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): except KeyError: return _not_found_error() else: - return reflection_pb2.ServerReflectionResponse( - all_extension_numbers_response=reflection_pb2. + return _reflection_pb2.ServerReflectionResponse( + all_extension_numbers_response=_reflection_pb2. ExtensionNumberResponse( base_type_name=message_descriptor.full_name, extension_number=extension_numbers)) def _list_services(self): - return reflection_pb2.ServerReflectionResponse( - list_services_response=reflection_pb2.ListServiceResponse( + return _reflection_pb2.ServerReflectionResponse( + list_services_response=_reflection_pb2.ListServiceResponse( service=[ - reflection_pb2.ServiceResponse(name=service_name) + _reflection_pb2.ServiceResponse(name=service_name) for service_name in self._service_names ])) @@ -126,8 +128,8 @@ class ReflectionServicer(reflection_pb2_grpc.ServerReflectionServicer): elif request.HasField('list_services'): yield self._list_services() else: - yield reflection_pb2.ServerReflectionResponse( - error_response=reflection_pb2.ErrorResponse( + yield _reflection_pb2.ServerReflectionResponse( + error_response=_reflection_pb2.ErrorResponse( error_code=grpc.StatusCode.INVALID_ARGUMENT.value[0], error_message=grpc.StatusCode.INVALID_ARGUMENT.value[1] .encode(), @@ -142,5 +144,5 @@ def enable_server_reflection(service_names, server, pool=None): server: grpc.Server to which reflection service will be added. pool: DescriptorPool object to use (descriptor_pool.Default() if None). """ - reflection_pb2_grpc.add_ServerReflectionServicer_to_server( + _reflection_pb2_grpc.add_ServerReflectionServicer_to_server( ReflectionServicer(service_names, pool=pool), server) diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 589d0ff5565..b4087d87b49 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -58,7 +58,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_testing/setup.py b/src/python/grpcio_testing/setup.py index eb480a5464a..6ceb1fc5c9d 100644 --- a/src/python/grpcio_testing/setup.py +++ b/src/python/grpcio_testing/setup.py @@ -29,7 +29,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 1262e485715..a94c0963ec1 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -41,8 +41,8 @@ INSTALL_REQUIRES = ( 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.5.2.post1', 'six>=1.10', - 'google-auth>=1.0.0', 'requests>=2.14.2') + 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', + 'requests>=2.14.2') if not PY3: INSTALL_REQUIRES += ('futures>=2.2.0',) diff --git a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py index 3cbbb8de33e..350b5eebe5b 100644 --- a/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py +++ b/src/python/grpcio_tests/tests/health_check/_health_servicer_test.py @@ -73,6 +73,9 @@ class HealthServicerTest(unittest.TestCase): self.assertEqual(grpc.StatusCode.NOT_FOUND, context.exception.code()) + def test_health_service_name(self): + self.assertEqual(health.SERVICE_NAME, 'grpc.health.v1.Health') + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py index 7ffdba6a677..bcd9e14a386 100644 --- a/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py +++ b/src/python/grpcio_tests/tests/reflection/_reflection_servicer_test.py @@ -171,6 +171,10 @@ class ReflectionServicerTest(unittest.TestCase): for name in _SERVICE_NAMES))),) self.assertSequenceEqual(expected_responses, responses) + def testReflectionServiceName(self): + self.assertEqual(reflection.SERVICE_NAME, + 'grpc.reflection.v1alpha.ServerReflection') + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py b/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py index aebee4da965..e0422627962 100644 --- a/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py +++ b/src/python/grpcio_tests/tests/unit/beta/_utilities_test.py @@ -41,6 +41,7 @@ class _Callback(object): return self._value +@unittest.skip('https://github.com/grpc/grpc/issues/16134') class ChannelConnectivityTest(unittest.TestCase): def test_lonely_channel_connectivity(self): diff --git a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel new file mode 100644 index 00000000000..d69186e1fde --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel @@ -0,0 +1,17 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "stream_testing", + srcs = ["stream_testing.py"], +) + +py_test( + name = "logging_pool_test", + srcs = ["_logging_pool_test.py"], + main = "_logging_pool_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 4760f33e389..505357021e0 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -15,31 +15,6 @@ require 'etc' require 'mkmf' -LIBDIR = RbConfig::CONFIG['libdir'] -INCLUDEDIR = RbConfig::CONFIG['includedir'] - -HEADER_DIRS = [ - # Search /opt/local (Mac source install) - '/opt/local/include', - - # Search /usr/local (Source install) - '/usr/local/include', - - # Check the ruby install locations - INCLUDEDIR -] - -LIB_DIRS = [ - # Search /opt/local (Mac source install) - '/opt/local/lib', - - # Search /usr/local (Source install) - '/usr/local/lib', - - # Check the ruby install locations - LIBDIR -] - windows = RUBY_PLATFORM =~ /mingw|mswin/ bsd = RUBY_PLATFORM =~ /bsd/ @@ -110,7 +85,7 @@ if grpc_config == 'opt' o.puts i end o.puts - o.puts 'strip:' + o.puts 'strip: $(DLLIB)' o.puts "\t$(ECHO) Stripping $(DLLIB)" o.puts "\t$(Q) #{strip_tool} $(DLLIB)" end diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 2443532bb8e..37b97aabd41 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -43,6 +43,7 @@ grpc_g_stands_for_type grpc_g_stands_for_import; grpc_completion_queue_factory_lookup_type grpc_completion_queue_factory_lookup_import; grpc_completion_queue_create_for_next_type grpc_completion_queue_create_for_next_import; grpc_completion_queue_create_for_pluck_type grpc_completion_queue_create_for_pluck_import; +grpc_completion_queue_create_for_callback_type grpc_completion_queue_create_for_callback_import; grpc_completion_queue_create_type grpc_completion_queue_create_import; grpc_completion_queue_next_type grpc_completion_queue_next_import; grpc_completion_queue_pluck_type grpc_completion_queue_pluck_import; @@ -65,6 +66,7 @@ grpc_census_call_set_context_type grpc_census_call_set_context_import; grpc_census_call_get_context_type grpc_census_call_get_context_import; grpc_channel_get_target_type grpc_channel_get_target_import; grpc_channel_get_info_type grpc_channel_get_info_import; +grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; grpc_insecure_channel_create_type grpc_insecure_channel_create_import; grpc_lame_client_channel_create_type grpc_lame_client_channel_create_import; grpc_channel_destroy_type grpc_channel_destroy_import; @@ -91,6 +93,7 @@ grpc_resource_quota_create_type grpc_resource_quota_create_import; grpc_resource_quota_ref_type grpc_resource_quota_ref_import; grpc_resource_quota_unref_type grpc_resource_quota_unref_import; grpc_resource_quota_resize_type grpc_resource_quota_resize_import; +grpc_resource_quota_set_max_threads_type grpc_resource_quota_set_max_threads_import; grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; grpc_channelz_get_top_channels_type grpc_channelz_get_top_channels_import; grpc_channelz_get_channel_type grpc_channelz_get_channel_import; @@ -293,6 +296,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_completion_queue_factory_lookup_import = (grpc_completion_queue_factory_lookup_type) GetProcAddress(library, "grpc_completion_queue_factory_lookup"); grpc_completion_queue_create_for_next_import = (grpc_completion_queue_create_for_next_type) GetProcAddress(library, "grpc_completion_queue_create_for_next"); grpc_completion_queue_create_for_pluck_import = (grpc_completion_queue_create_for_pluck_type) GetProcAddress(library, "grpc_completion_queue_create_for_pluck"); + grpc_completion_queue_create_for_callback_import = (grpc_completion_queue_create_for_callback_type) GetProcAddress(library, "grpc_completion_queue_create_for_callback"); grpc_completion_queue_create_import = (grpc_completion_queue_create_type) GetProcAddress(library, "grpc_completion_queue_create"); grpc_completion_queue_next_import = (grpc_completion_queue_next_type) GetProcAddress(library, "grpc_completion_queue_next"); grpc_completion_queue_pluck_import = (grpc_completion_queue_pluck_type) GetProcAddress(library, "grpc_completion_queue_pluck"); @@ -315,6 +319,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_census_call_get_context_import = (grpc_census_call_get_context_type) GetProcAddress(library, "grpc_census_call_get_context"); grpc_channel_get_target_import = (grpc_channel_get_target_type) GetProcAddress(library, "grpc_channel_get_target"); grpc_channel_get_info_import = (grpc_channel_get_info_type) GetProcAddress(library, "grpc_channel_get_info"); + grpc_channel_reset_connect_backoff_import = (grpc_channel_reset_connect_backoff_type) GetProcAddress(library, "grpc_channel_reset_connect_backoff"); grpc_insecure_channel_create_import = (grpc_insecure_channel_create_type) GetProcAddress(library, "grpc_insecure_channel_create"); grpc_lame_client_channel_create_import = (grpc_lame_client_channel_create_type) GetProcAddress(library, "grpc_lame_client_channel_create"); grpc_channel_destroy_import = (grpc_channel_destroy_type) GetProcAddress(library, "grpc_channel_destroy"); @@ -341,6 +346,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_resource_quota_ref_import = (grpc_resource_quota_ref_type) GetProcAddress(library, "grpc_resource_quota_ref"); grpc_resource_quota_unref_import = (grpc_resource_quota_unref_type) GetProcAddress(library, "grpc_resource_quota_unref"); grpc_resource_quota_resize_import = (grpc_resource_quota_resize_type) GetProcAddress(library, "grpc_resource_quota_resize"); + grpc_resource_quota_set_max_threads_import = (grpc_resource_quota_set_max_threads_type) GetProcAddress(library, "grpc_resource_quota_set_max_threads"); grpc_resource_quota_arg_vtable_import = (grpc_resource_quota_arg_vtable_type) GetProcAddress(library, "grpc_resource_quota_arg_vtable"); grpc_channelz_get_top_channels_import = (grpc_channelz_get_top_channels_type) GetProcAddress(library, "grpc_channelz_get_top_channels"); grpc_channelz_get_channel_import = (grpc_channelz_get_channel_type) GetProcAddress(library, "grpc_channelz_get_channel"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index b08a1f94f7b..f7a00046e31 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -104,6 +104,9 @@ extern grpc_completion_queue_create_for_next_type grpc_completion_queue_create_f typedef grpc_completion_queue*(*grpc_completion_queue_create_for_pluck_type)(void* reserved); extern grpc_completion_queue_create_for_pluck_type grpc_completion_queue_create_for_pluck_import; #define grpc_completion_queue_create_for_pluck grpc_completion_queue_create_for_pluck_import +typedef grpc_completion_queue*(*grpc_completion_queue_create_for_callback_type)(void* shutdown_callback, void* reserved); +extern grpc_completion_queue_create_for_callback_type grpc_completion_queue_create_for_callback_import; +#define grpc_completion_queue_create_for_callback grpc_completion_queue_create_for_callback_import typedef grpc_completion_queue*(*grpc_completion_queue_create_type)(const grpc_completion_queue_factory* factory, const grpc_completion_queue_attributes* attributes, void* reserved); extern grpc_completion_queue_create_type grpc_completion_queue_create_import; #define grpc_completion_queue_create grpc_completion_queue_create_import @@ -170,6 +173,9 @@ extern grpc_channel_get_target_type grpc_channel_get_target_import; typedef void(*grpc_channel_get_info_type)(grpc_channel* channel, const grpc_channel_info* channel_info); extern grpc_channel_get_info_type grpc_channel_get_info_import; #define grpc_channel_get_info grpc_channel_get_info_import +typedef void(*grpc_channel_reset_connect_backoff_type)(grpc_channel* channel); +extern grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; +#define grpc_channel_reset_connect_backoff grpc_channel_reset_connect_backoff_import typedef grpc_channel*(*grpc_insecure_channel_create_type)(const char* target, const grpc_channel_args* args, void* reserved); extern grpc_insecure_channel_create_type grpc_insecure_channel_create_import; #define grpc_insecure_channel_create grpc_insecure_channel_create_import @@ -248,6 +254,9 @@ extern grpc_resource_quota_unref_type grpc_resource_quota_unref_import; typedef void(*grpc_resource_quota_resize_type)(grpc_resource_quota* resource_quota, size_t new_size); extern grpc_resource_quota_resize_type grpc_resource_quota_resize_import; #define grpc_resource_quota_resize grpc_resource_quota_resize_import +typedef void(*grpc_resource_quota_set_max_threads_type)(grpc_resource_quota* resource_quota, int new_max_threads); +extern grpc_resource_quota_set_max_threads_type grpc_resource_quota_set_max_threads_import; +#define grpc_resource_quota_set_max_threads grpc_resource_quota_set_max_threads_import typedef const grpc_arg_pointer_vtable*(*grpc_resource_quota_arg_vtable_type)(void); extern grpc_resource_quota_arg_vtable_type grpc_resource_quota_arg_vtable_import; #define grpc_resource_quota_arg_vtable grpc_resource_quota_arg_vtable_import diff --git a/summerofcode/2018/naresh.md b/summerofcode/2018/naresh.md new file mode 100644 index 00000000000..0d196bd6001 --- /dev/null +++ b/summerofcode/2018/naresh.md @@ -0,0 +1,191 @@ +# Project overview + +## Title + +Enable Building of gRPC Python with Bazel + +## Overview + +gRPC Python currently has a constellation of scripts written to build the +project, but it has a lot of limitations in terms of speed and maintainability. +[Bazel](https://bazel.build/) is the open-sourced variant of Google's internal +system, Blaze, which is an ideal replacement for building such projects in a +fast and declarative fashion. But Bazel in itself is still in active +development, especially in terms of Python (amongst a few other languages). + +The project aimed to fill this gap and build gRPC Python with Bazel. + +[Project page](https://summerofcode.withgoogle.com/projects/#6482576244473856) + +[Link to proposal](https://storage.googleapis.com/summerofcode-prod.appspot.com/gsoc/core_project/doc/5316764725411840_1522049732_Naresh_Ramesh_-_GSoC_proposal.pdf) + +## Thoughts and challenges + +### State of Bazel for Python + +Although previously speculated, the project didn't require any contributions +directly to [bazelbuild/bazel](https://github.com/bazelbuild/bazel). The Bazel +rules for Python are currently being separated out into their own repo at +[bazelbuild/rules_python](https://github.com/bazelbuild/rules_python/). + +Bazel is [still very much in active development for +Python](https://groups.google.com/forum/#!topic/bazel-sig-python/iQjV9sfSufw) +though. There's still challenges when it comes to building for Python 2 vs 3. +Using pip packages is still in experimental. Bazel Python support is currently +distributed across these two repositories and is yet to begin migration to one +place (which will be +[bazelbuild/rules_python](https://github.com/bazelbuild/rules_python/)). + +Bazel's roadmap for Python is publicly available [here as a Google +doc](https://docs.google.com/document/d/1A6J3j3y1SQ0HliS86_mZBnB5UeBe7vExWL2Ryd_EONI/edit). + +### Cross collaboration between projects + +Cross contribution surprisingly came up because of building protobuf sources +for Python, which is still not natively supported by Bazel. An existing +repository, [pubref/rules_protobuf](https://github.com/pubref/rules_protobuf), +which was maintained by an independent maintainer (i.e. not a part of Bazel) +helped solve this problem, but had [one major blocking +issue](https://github.com/pubref/rules_protobuf/issues/233) and could not be +resolved at the source. But [a solution to the +issue](https://github.com/pubref/rules_protobuf/pull/196) was proposed by user +dududko, which was not merged because of failing golang tests but worked well +for Python. Hence, a fork of this repo was made and is to be used with gRPC +until the solution can be merged back at the source. + +### Building Cython code + +Building Cython code is still not supported by Bazel, but the team at +[cython/cython](https://github.com/cython/cython) have added support for Bazel +on their side. The way it works is by including Cython as a third-party Bazel +dependency and using custom Bazel rules for building our Cython code using the +binary within the dependency. + +### Packaging Python code using Bazel + +pip and PyPI still remain the de-facto standard for distributing Python +packages. Although Bazel is pretty versatile and is amazing for it's +reproducible and incremental build capabilities, these can only be still used +by the contributors and developers for building and testing the gRPC code. But +there's no way yet to build Python packages for distribution. + +### Building gRPC Python with Bazel on Kokoro (internal CI) + +Integration with the internal CI was one of the areas that highlighted how +simple Bazel can be to use. gRPC was already using a dockerized Bazel setup to +build some of it's core code (but not as the primary build setup). Adding a new +job on the internal CI ended up being as simple as creating a new shell script +to install the required dependencies (which were python-dev and Bazel) and a +new configuration file which pointed to the subdirectiory (src/python) under +which to look for targets and run the tests accordingly. + +### Handling imports in Python code + +When writing Python packages, imports in nested modules are typically made +relative to the package root. But because of the way Bazel works, these paths +wouldn't make sense from the Workspace root. So, the folks at Bazel have added +a nifty `imports` parameter to all the Python rules which lets us specify for +each target, which path to consider as the root. This parameter allows for +relative paths like `imports = ["../",]`. + +### Fetching Python headers for Cython code to use + +Cython code makes use of `Python.h`, which pulls in the Python API for C +extension modules to use, but it's location depending on the Python version and +operating system the code is building on. To make this easier, the folks at +Tensorflow wrote [repository rules for Python +autoconfiguration](https://github.com/tensorflow/tensorflow/tree/e447ae4759317156d31a9421290716f0ffbffcd8/third_party/py). +This has been [adapted with some some +modifications](https://github.com/grpc/grpc/pull/15992) for use in gRPC Python +as well. + +## How to use + +All the Bazel tests for gRPC Python can be run using a single command: + +```bash +bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/... +``` + +If any specific test is to be run, like say `LoggingPoolTest` (which is present +in +`src/python/grpcio_tests/tests/unit/framework/foundation/_logging_pool_test.py`), +the command to run would be: + +```bash +bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/grpcio_tests/tests/unit/framework/foundation:logging_pool_test +``` + +where, `logging_pool_test` is the name of the Bazel target for this test. + +Similarly, to run a particular method, use: + +```bash +bazel test --spawn_strategy=standalone --genrule_strategy=standalone //src/python/grpcio_tests/tests/unit/_rpc_test --test_arg=RPCTest.testUnrecognizedMethod +``` + +## Useful Bazel flags + +- Use `bazel build` with a `-s` flag to see the logs being printed out to + standard output while building. +- Similarly, use `bazel test` with a `--test_output=streamed` to see the the + test logs while testing. Something to know while using this flag is that all + tests will be run locally, without sharding, one at a time. + +## Contributions + +### Related to the project + +- [435c6f8](https://github.com/grpc/grpc/commit/435c6f8d1e53783ec049b3482445813afd8bc514) + Update grpc_gevent cython files to include .pxi +- [74426fd](https://github.com/grpc/grpc/commit/74426fd2164c51d6754732ebe372133c19ba718c) + Add gevent_util.h to grpc_base_c Bazel target +- [b6518af](https://github.com/grpc/grpc/commit/b6518afdd610f0115b42aee1ffc71520c6b0d6b1) + Upgrade Bazel to 0.15.0 +- [ebcf04d](https://github.com/grpc/grpc/commit/ebcf04d075333c42979536c5dd2091d363f67e5a) + Kokoro setup for building gRPC Python with Bazel +- [3af1aaa](https://github.com/grpc/grpc/commit/3af1aaadabf49bc6274711a11f81627c0f351a9a) + Basic setup to build gRPC Python with Bazel +- [11f199e](https://github.com/grpc/grpc/commit/11f199e34dc416a2bd8b56391b242a867bedade4) + Workspace changes to build gRPC Python with Bazel +- [848fd9d](https://github.com/grpc/grpc/commit/848fd9d75f6df10f00e8328ff052c0237b3002ab) + Minimal Bazel BUILD files for grpcio Python + +### Other contibutions + +- [89ce16b](https://github.com/grpc/grpc/commit/89ce16b6daaad4caeb1c9ba670c6c4b62ea1a93c) + Update Dockerfiles for python artifacts to use latest git version +- [32f7c48](https://github.com/grpc/grpc/commit/32f7c48dad71cac7af652bf994ab1dde3ddb0607) + Revert removals from python artifact dockerfiles +- [712eb9f](https://github.com/grpc/grpc/commit/712eb9ff91cde66af94e8381ec01ad512ed6d03c) + Make logging after success in jobset more apparent +- [c6e4372](https://github.com/grpc/grpc/commit/c6e4372f8a93bb0eb996b5f202465785422290f2) + Create README for gRPC Python reflection package +- [2e113ca](https://github.com/grpc/grpc/commit/2e113ca6b2cc31aa8a9687d40ee1bd759381654f) + Update logging in Python to use module-level logger + +### Pending PRs + +- BUILD files for all tests in + [tests.json](https://github.com/ghostwriternr/grpc/blob/70c8a58b2918a5369905e5a203d7ce7897b6207e/src/python/grpcio_tests/tests/tests.json). +- BUILD files for gRPC testing, gRPC health checking, gRPC reflection. +- (Yet to complete) BUILD files for grpcio_tools. One test depends on this. + +## Known issues + +- [grpc/grpc #16336](https://github.com/grpc/grpc/issues/16336) RuntimeError + for `_reconnect_test` Python unit test with Bazel +- Some tests in Bazel pass despite throwing an exception. Example: + `testAbortedStreamStream` in + `src/python/grpcio_tests/tests/unit/_metadata_code_details_test.py`. +- [#14557](https://github.com/grpc/grpc/pull/14557) introduced a minor bug + where the module level loggers don't initialize a default logging handler. +- Sanity test doesn't make sense in the context of Bazel, and thus fails. +- There are some issues with Python2 vs Python3. Specifically, + - On some machines, “cygrpc.so: undefined symbol: _Py_FalseStruct” error + shows up. This is because of incorrect Python version being used to build + Cython. + - Some external packages like enum34 throw errors when used with Python 3 and + some extra packages are currently installed as Python version in current + build scripts. For now, the extra packages are added to a + `requirements.bazel.txt` file in the repository root. diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index de13d02e2a3..2a514ed7b68 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -149,6 +149,8 @@ add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later diff --git a/templates/Makefile.template b/templates/Makefile.template index 628056ed4db..2e3d75d819e 100644 --- a/templates/Makefile.template +++ b/templates/Makefile.template @@ -668,11 +668,20 @@ LDLIBS_SECURE += $(addprefix -l, $(LIBS_SECURE)) endif + # gpr .pc file + PC_NAME = gpr + PC_DESCRIPTION = gRPC platform support library + PC_CFLAGS = + PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GPR) + PC_LIBS_PRIVATE = $(PC_LIBS_GPR) + PC_LIB = -lgpr + GPR_PC_FILE := $(CORE_PC_TEMPLATE) + # grpc .pc file PC_NAME = gRPC PC_DESCRIPTION = high performance general RPC framework PC_CFLAGS = - PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GRPC) $(PC_REQUIRES_SECURE) + PC_REQUIRES_PRIVATE = gpr $(PC_REQUIRES_GRPC) $(PC_REQUIRES_SECURE) PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) $(PC_LIBS_SECURE) PC_LIB = -lgrpc GRPC_PC_FILE := $(CORE_PC_TEMPLATE) @@ -681,7 +690,7 @@ PC_NAME = gRPC unsecure PC_DESCRIPTION = high performance general RPC framework without SSL PC_CFLAGS = - PC_REQUIRES_PRIVATE = $(PC_REQUIRES_GRPC) + PC_REQUIRES_PRIVATE = gpr $(PC_REQUIRES_GRPC) PC_LIBS_PRIVATE = $(PC_LIBS_GRPC) PC_LIB = -lgrpc GRPC_UNSECURE_PC_FILE := $(CORE_PC_TEMPLATE) @@ -921,6 +930,16 @@ % endfor + static_csharp: static_c \ + % for lib in libs: + % if 'Makefile' in lib.get('build_system', ['Makefile']): + % if lib.build == 'all' and lib.language == 'csharp': + $(LIBDIR)/$(CONFIG)/lib${lib.name}.a\ + % endif + % endif + % endfor + + shared: shared_c shared_cxx shared_c: pc_c pc_c_unsecure cache.mk\ @@ -966,9 +985,9 @@ % endif % endfor - pc_c: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc + pc_c: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc - pc_c_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc + pc_c_unsecure: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc pc_cxx: $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc @@ -1189,6 +1208,11 @@ $(E) "[MAKE] Generating $@" $(Q) echo "$(CACHE_MK)" | tr , '\n' >$@ + $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc: + $(E) "[MAKE] Generating $@" + $(Q) mkdir -p $(@D) + $(Q) echo "$(GPR_PC_FILE)" | tr , '\n' >$@ + $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc: $(E) "[MAKE] Generating $@" $(Q) mkdir -p $(@D) @@ -1387,6 +1411,7 @@ install-pkg-config_c: pc_c pc_c_unsecure $(E) "[INSTALL] Installing C pkg-config files" $(Q) $(INSTALL) -d $(prefix)/lib/pkgconfig + $(Q) $(INSTALL) -m 0644 $(LIBDIR)/$(CONFIG)/pkgconfig/gpr.pc $(prefix)/lib/pkgconfig/gpr.pc $(Q) $(INSTALL) -m 0644 $(LIBDIR)/$(CONFIG)/pkgconfig/grpc.pc $(prefix)/lib/pkgconfig/grpc.pc $(Q) $(INSTALL) -m 0644 $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc $(prefix)/lib/pkgconfig/grpc_unsecure.pc diff --git a/templates/composer.json.template b/templates/composer.json.template index aa6cbb8bcb2..a9cc6cca865 100644 --- a/templates/composer.json.template +++ b/templates/composer.json.template @@ -11,7 +11,7 @@ "php": ">=5.5.0" }, "require-dev": { - "google/auth": "v0.9" + "google/auth": "^v1.3.0" }, "suggest": { "ext-protobuf": "For better performance, install the protobuf C extension.", diff --git a/templates/src/csharp/Grpc.Core/Version.csproj.include.template b/templates/src/csharp/Grpc.Core/Version.csproj.include.template index 398b198dbd1..0ec0a08c499 100755 --- a/templates/src/csharp/Grpc.Core/Version.csproj.include.template +++ b/templates/src/csharp/Grpc.Core/Version.csproj.include.template @@ -4,6 +4,6 @@ ${settings.csharp_version} - 3.5.1 + 3.6.1 diff --git a/templates/src/csharp/build_packages_dotnetcli.bat.template b/templates/src/csharp/build_packages_dotnetcli.bat.template index 45010fec741..2ae909f53af 100755 --- a/templates/src/csharp/build_packages_dotnetcli.bat.template +++ b/templates/src/csharp/build_packages_dotnetcli.bat.template @@ -23,18 +23,12 @@ mkdir ..\..\artifacts - @rem Collect the artifacts built by the previous build step if running on Jenkins + @rem Collect the artifacts built by the previous build step mkdir nativelibs - @rem Jenkins flow (deprecated) - powershell -Command "cp -r ..\..\platform=*\artifacts\csharp_ext_* nativelibs" - @rem Kokoro flow powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs" @rem Collect protoc artifacts built by the previous build step mkdir protoc_plugins - @rem Jenkins flow (deprecated) - powershell -Command "cp -r ..\..\platform=*\artifacts\protoc_* protoc_plugins" - @rem Kokoro flow powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins" %%DOTNET% restore Grpc.sln || goto :error diff --git a/templates/src/csharp/build_packages_dotnetcli.sh.template b/templates/src/csharp/build_packages_dotnetcli.sh.template deleted file mode 100755 index 1172582ebde..00000000000 --- a/templates/src/csharp/build_packages_dotnetcli.sh.template +++ /dev/null @@ -1,54 +0,0 @@ -%YAML 1.2 ---- | - #!/bin/bash - # Copyright 2016 gRPC authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - set -ex - - cd $(dirname $0) - - mkdir -p ../../artifacts/ - - # Collect the artifacts built by the previous build step - mkdir -p nativelibs - # Jenkins flow (deprecated) - cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/csharp_ext_* nativelibs || true - # Kokoro flow - cp -r $EXTERNAL_GIT_ROOT/input_artifacts/csharp_ext_* nativelibs || true - - # Collect protoc artifacts built by the previous build step - mkdir -p protoc_plugins - # Jenkins flow (deprecated) - cp -r $EXTERNAL_GIT_ROOT/platform={windows,linux,macos}/artifacts/protoc_* protoc_plugins || true - # Kokoro flow - cp -r $EXTERNAL_GIT_ROOT/input_artifacts/protoc_* protoc_plugins || true - - dotnet restore Grpc.sln - - # To be able to build, we also need to put grpc_csharp_ext to its normal location - mkdir -p ../../libs/opt - cp nativelibs/csharp_ext_linux_x64/libgrpc_csharp_ext.so ../../libs/opt - - dotnet pack --configuration Release Grpc.Core --output ../../../artifacts - dotnet pack --configuration Release Grpc.Core.Testing --output ../../../artifacts - dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts - dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts - dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts - - nuget pack Grpc.nuspec -Version "${settings.csharp_version}" -OutputDirectory ../../artifacts - nuget pack Grpc.Core.NativeDebug.nuspec -Version "${settings.csharp_version}" -OutputDirectory ../../artifacts - nuget pack Grpc.Tools.nuspec -Version "${settings.csharp_version}" -OutputDirectory ../../artifacts - - (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/templates/src/csharp/build_unitypackage.bat.template b/templates/src/csharp/build_unitypackage.bat.template new file mode 100755 index 00000000000..76ec10dbd90 --- /dev/null +++ b/templates/src/csharp/build_unitypackage.bat.template @@ -0,0 +1,82 @@ +%YAML 1.2 +--- | + @rem Copyright 2018 The gRPC Authors + @rem + @rem Licensed under the Apache License, Version 2.0 (the "License"); + @rem you may not use this file except in compliance with the License. + @rem You may obtain a copy of the License at + @rem + @rem http://www.apache.org/licenses/LICENSE-2.0 + @rem + @rem Unless required by applicable law or agreed to in writing, software + @rem distributed under the License is distributed on an "AS IS" BASIS, + @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + @rem See the License for the specific language governing permissions and + @rem limitations under the License. + + @rem Current package versions + set VERSION=${settings.csharp_version} + + @rem Adjust the location of nuget.exe + set NUGET=C:\nuget\nuget.exe + set DOTNET=dotnet + + mkdir ..\..\artifacts + + @rem Collect the artifacts built by the previous build step + mkdir nativelibs + powershell -Command "cp -r ..\..\input_artifacts\csharp_ext_* nativelibs" + + @rem Collect protoc artifacts built by the previous build step + mkdir protoc_plugins + powershell -Command "cp -r ..\..\input_artifacts\protoc_* protoc_plugins" + + %%DOTNET% restore Grpc.sln || goto :error + + @rem To be able to build, we also need to put grpc_csharp_ext to its normal location + xcopy /Y /I nativelibs\csharp_ext_windows_x64\grpc_csharp_ext.dll ..\..\cmake\build\x64\Release${"\\"} + + %%DOTNET% build --configuration Release Grpc.Core || goto :error + @rem build HealthCheck to get hold of Google.Protobuf.dll assembly + %%DOTNET% build --configuration Release Grpc.HealthCheck || goto :error + + @rem copy Grpc assemblies to the unity package skeleton + @rem TODO(jtattermusch): Add Grpc.Auth assembly and its dependencies + copy /Y Grpc.Core\bin\Release\net45\Grpc.Core.dll unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\lib\net45\Grpc.Core.dll || goto :error + copy /Y Grpc.Core\bin\Release\net45\Grpc.Core.pdb unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\lib\net45\Grpc.Core.pdb || goto :error + copy /Y Grpc.Core\bin\Release\net45\Grpc.Core.xml unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\lib\net45\Grpc.Core.xml || goto :error + + @rem copy desktop native libraries to the unity package skeleton + copy /Y nativelibs\csharp_ext_linux_x86\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\linux\x86\libgrpc_csharp_ext.so || goto :error + copy /Y nativelibs\csharp_ext_linux_x64\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\linux\x64\libgrpc_csharp_ext.so || goto :error + copy /Y nativelibs\csharp_ext_macos_x86\libgrpc_csharp_ext.dylib unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\osx\x86\grpc_csharp_ext.bundle || goto :error + copy /Y nativelibs\csharp_ext_macos_x64\libgrpc_csharp_ext.dylib unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\osx\x64\grpc_csharp_ext.bundle || goto :error + copy /Y nativelibs\csharp_ext_windows_x86\grpc_csharp_ext.dll unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\win\x86\grpc_csharp_ext.dll || goto :error + copy /Y nativelibs\csharp_ext_windows_x64\grpc_csharp_ext.dll unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\win\x64\grpc_csharp_ext.dll || goto :error + + @rem add Android and iOS native libraries + copy /Y nativelibs\csharp_ext_linux_android_armeabi-v7a\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\android\armeabi-v7a\libgrpc_csharp_ext.so || goto :error + copy /Y nativelibs\csharp_ext_linux_android_arm64-v8a\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\android\arm64-v8a\libgrpc_csharp_ext.so || goto :error + copy /Y nativelibs\csharp_ext_linux_android_x86\libgrpc_csharp_ext.so unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\android\x86\libgrpc_csharp_ext.so || goto :error + copy /Y nativelibs\csharp_ext_macos_ios\libgrpc_csharp_ext.a unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\ios\libgrpc_csharp_ext.a || goto :error + copy /Y nativelibs\csharp_ext_macos_ios\libgrpc.a unitypackage\unitypackage_skeleton\Plugins\Grpc.Core\runtimes\ios\libgrpc.a || goto :error + + @rem add gRPC dependencies + @rem TODO(jtattermusch): also include XMLdoc + copy /Y Grpc.Core\bin\Release\net45\System.Interactive.Async.dll unitypackage\unitypackage_skeleton\Plugins\System.Interactive.Async\lib\net45\System.Interactive.Async.dll || goto :error + + @rem add Google.Protobuf + @rem TODO(jtattermusch): also include XMLdoc + copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error + + @rem create a zipfile that will act as a Unity package + cd unitypackage\unitypackage_skeleton + zip -r ..\..\grpc_unity_package.zip Plugins + cd ..\.. + copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error + + goto :EOF + + :error + echo Failed! + exit /b %errorlevel% diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index 5c1358f7c37..30b6c5684cc 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/templates/src/php/composer.json.template b/templates/src/php/composer.json.template index d647997113e..3793634a2fd 100644 --- a/templates/src/php/composer.json.template +++ b/templates/src/php/composer.json.template @@ -10,7 +10,7 @@ "google/protobuf": "^v3.3.0" }, "require-dev": { - "google/auth": "v0.9" + "google/auth": "^v1.3.0" }, "autoload": { "psr-4": { diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include index bc981dc83e1..b34845e01a3 100644 --- a/templates/test/cpp/naming/resolver_component_tests_defs.include +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) diff --git a/test/core/channel/channel_trace_test.cc b/test/core/channel/channel_trace_test.cc index 99d9a4847fd..f224457a557 100644 --- a/test/core/channel/channel_trace_test.cc +++ b/test/core/channel/channel_trace_test.cc @@ -187,8 +187,8 @@ TEST_P(ChannelTracerTest, ComplexTest) { AddSimpleTrace(&tracer); AddSimpleTrace(&tracer); AddSimpleTrace(&tracer); - sc1.reset(nullptr); - sc2.reset(nullptr); + sc1.reset(); + sc2.reset(); } // Test a case in which the parent channel has subchannels and the subchannels @@ -234,9 +234,9 @@ TEST_P(ChannelTracerTest, TestNesting) { grpc_slice_from_static_string("subchannel one inactive"), sc1); AddSimpleTrace(&tracer); ValidateChannelTrace(&tracer, 8, GetParam()); - sc1.reset(nullptr); - sc2.reset(nullptr); - conn1.reset(nullptr); + sc1.reset(); + sc2.reset(); + conn1.reset(); } INSTANTIATE_TEST_CASE_P(ChannelTracerTestSweep, ChannelTracerTest, diff --git a/test/core/gprpp/ref_counted_ptr_test.cc b/test/core/gprpp/ref_counted_ptr_test.cc index aa30b72282f..463b5e89667 100644 --- a/test/core/gprpp/ref_counted_ptr_test.cc +++ b/test/core/gprpp/ref_counted_ptr_test.cc @@ -127,7 +127,7 @@ TEST(RefCountedPtr, ResetFromNonNullToNull) { TEST(RefCountedPtr, ResetFromNullToNull) { RefCountedPtr foo; EXPECT_EQ(nullptr, foo.get()); - foo.reset(nullptr); + foo.reset(); EXPECT_EQ(nullptr, foo.get()); } @@ -175,6 +175,67 @@ TEST(RefCountedPtr, RefCountedWithTracing) { foo->Unref(DEBUG_LOCATION, "foo"); } +class BaseClass : public RefCounted { + public: + BaseClass() {} +}; + +class Subclass : public BaseClass { + public: + Subclass() {} +}; + +TEST(RefCountedPtr, ConstructFromSubclass) { + RefCountedPtr p(New()); +} + +TEST(RefCountedPtr, CopyAssignFromSubclass) { + RefCountedPtr b; + EXPECT_EQ(nullptr, b.get()); + RefCountedPtr s = MakeRefCounted(); + b = s; + EXPECT_NE(nullptr, b.get()); +} + +TEST(RefCountedPtr, MoveAssignFromSubclass) { + RefCountedPtr b; + EXPECT_EQ(nullptr, b.get()); + RefCountedPtr s = MakeRefCounted(); + b = std::move(s); + EXPECT_NE(nullptr, b.get()); +} + +TEST(RefCountedPtr, ResetFromSubclass) { + RefCountedPtr b; + EXPECT_EQ(nullptr, b.get()); + b.reset(New()); + EXPECT_NE(nullptr, b.get()); +} + +TEST(RefCountedPtr, EqualityWithSubclass) { + Subclass* s = New(); + RefCountedPtr b(s); + EXPECT_EQ(b, s); +} + +void FunctionTakingBaseClass(RefCountedPtr p) { + p.reset(); // To appease clang-tidy. +} + +TEST(RefCountedPtr, CanPassSubclassToFunctionExpectingBaseClass) { + RefCountedPtr p = MakeRefCounted(); + FunctionTakingBaseClass(p); +} + +void FunctionTakingSubclass(RefCountedPtr p) { + p.reset(); // To appease clang-tidy. +} + +TEST(RefCountedPtr, CanPassSubclassToFunctionExpectingSubclass) { + RefCountedPtr p = MakeRefCounted(); + FunctionTakingSubclass(p); +} + } // namespace } // namespace testing } // namespace grpc_core diff --git a/test/core/iomgr/BUILD b/test/core/iomgr/BUILD index fb0490a95f4..002671a5fae 100644 --- a/test/core/iomgr/BUILD +++ b/test/core/iomgr/BUILD @@ -124,6 +124,19 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "grpc_ipv6_loopback_available_test", + srcs = ["grpc_ipv6_loopback_available_test.cc"], + language = "C++", + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + + grpc_cc_test( name = "load_file_test", srcs = ["load_file_test.cc"], diff --git a/test/core/iomgr/grpc_ipv6_loopback_available_test.cc b/test/core/iomgr/grpc_ipv6_loopback_available_test.cc new file mode 100644 index 00000000000..329aa9a8517 --- /dev/null +++ b/test/core/iomgr/grpc_ipv6_loopback_available_test.cc @@ -0,0 +1,48 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include "src/core/lib/iomgr/port.h" + +// grpc_ipv6_loopback_available isn't currently available on UV. +#ifndef GRPC_UV + +#include +#include +#include "test/core/util/test_config.h" + +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/socket_windows.h" +#else +#include "src/core/lib/iomgr/socket_utils_posix.h" +#endif + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + grpc_init(); + // This test assumes that the ipv6 loopback is available + // in all environments in which grpc tests run in. + GPR_ASSERT(grpc_ipv6_loopback_available()); + grpc_shutdown(); + return 0; +} + +#else + +int main(int argc, char** argv) { return 0; } + +#endif /* GRPC_UV */ diff --git a/test/core/iomgr/resource_quota_test.cc b/test/core/iomgr/resource_quota_test.cc index 059ff7b5f8b..f3b35fed327 100644 --- a/test/core/iomgr/resource_quota_test.cc +++ b/test/core/iomgr/resource_quota_test.cc @@ -798,6 +798,98 @@ static void test_negative_rq_free_pool(void) { } } +// Simple test to check resource quota thread limits +static void test_thread_limit() { + grpc_core::ExecCtx exec_ctx; + + grpc_resource_quota* rq = grpc_resource_quota_create("test_thread_limit"); + grpc_resource_user* ru1 = grpc_resource_user_create(rq, "ru1"); + grpc_resource_user* ru2 = grpc_resource_user_create(rq, "ru2"); + + // Max threads = 100 + grpc_resource_quota_set_max_threads(rq, 100); + + // Request quota for 100 threads (50 for ru1, 50 for ru2) + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 10)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 10)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 40)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 40)); + + // Threads exhausted. Next request must fail + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru2, 20)); + + // Free 20 threads from two different users + grpc_resource_user_free_threads(ru1, 10); + grpc_resource_user_free_threads(ru2, 10); + + // Next request to 20 threads must succeed + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 20)); + + // No more thread quota again + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 20)); + + // Free 10 more + grpc_resource_user_free_threads(ru1, 10); + + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 5)); + GPR_ASSERT( + !grpc_resource_user_allocate_threads(ru2, 10)); // Only 5 available + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 5)); + + // Teardown (ru1 and ru2 release all the quota back to rq) + grpc_resource_user_unref(ru1); + grpc_resource_user_unref(ru2); + grpc_resource_quota_unref(rq); +} + +// Change max quota in either direction dynamically +static void test_thread_maxquota_change() { + grpc_core::ExecCtx exec_ctx; + + grpc_resource_quota* rq = + grpc_resource_quota_create("test_thread_maxquota_change"); + grpc_resource_user* ru1 = grpc_resource_user_create(rq, "ru1"); + grpc_resource_user* ru2 = grpc_resource_user_create(rq, "ru2"); + + // Max threads = 100 + grpc_resource_quota_set_max_threads(rq, 100); + + // Request quota for 100 threads (50 for ru1, 50 for ru2) + GPR_ASSERT(grpc_resource_user_allocate_threads(ru1, 50)); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 50)); + + // Threads exhausted. Next request must fail + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru2, 20)); + + // Increase maxquota and retry + // Max threads = 150; + grpc_resource_quota_set_max_threads(rq, 150); + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 20)); // ru2=70, ru1=50 + + // Decrease maxquota (Note: Quota already given to ru1 and ru2 is unaffected) + // Max threads = 10; + grpc_resource_quota_set_max_threads(rq, 10); + + // New requests will fail until quota is available + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 10)); + + // Make quota available + grpc_resource_user_free_threads(ru1, 50); // ru1 now has 0 + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 10)); // not enough + + grpc_resource_user_free_threads(ru2, 70); // ru2 now has 0 + + // Now we can get quota up-to 10, the current max + GPR_ASSERT(grpc_resource_user_allocate_threads(ru2, 10)); + // No more thread quota again + GPR_ASSERT(!grpc_resource_user_allocate_threads(ru1, 10)); + + // Teardown (ru1 and ru2 release all the quota back to rq) + grpc_resource_user_unref(ru1); + grpc_resource_user_unref(ru2); + grpc_resource_quota_unref(rq); +} + int main(int argc, char** argv) { grpc_test_init(argc, argv); grpc_init(); @@ -827,6 +919,11 @@ int main(int argc, char** argv) { test_negative_rq_free_pool(); gpr_mu_destroy(&g_mu); gpr_cv_destroy(&g_cv); + + // Resource quota thread related + test_thread_limit(); + test_thread_maxquota_change(); + grpc_shutdown(); return 0; } diff --git a/test/core/security/BUILD b/test/core/security/BUILD index 12aa84d93b3..b7de955cdbe 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -128,6 +128,27 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "linux_system_roots_test", + srcs = ["linux_system_roots_test.cc"], + data = [ + "//test/core/security/etc:bundle.pem", + "//test/core/security/etc:test_roots/cert1.pem", + "//test/core/security/etc:test_roots/cert2.pem", + "//test/core/security/etc:test_roots/cert3.pem", + ], + language = "C++", + external_deps = [ + "gtest", + ], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "ssl_credentials_test", srcs = ["ssl_credentials_test.cc"], @@ -219,9 +240,9 @@ grpc_cc_test( deps = [ "//:gpr", "//:grpc", - "//:grpc_base_c", + "//:grpc_base_c", "//:grpc_secure", - "//:tsi", + "//:tsi", "//:tsi_interface", "//test/core/util:gpr_test_util", ], diff --git a/test/core/security/etc/BUILD b/test/core/security/etc/BUILD new file mode 100644 index 00000000000..2c6ab64a3b8 --- /dev/null +++ b/test/core/security/etc/BUILD @@ -0,0 +1,22 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache v2 + +exports_files([ + "bundle.pem", + "test_roots/cert1.pem", + "test_roots/cert2.pem", + "test_roots/cert3.pem", +]) diff --git a/test/core/security/etc/README b/test/core/security/etc/README new file mode 100644 index 00000000000..6ba43825861 --- /dev/null +++ b/test/core/security/etc/README @@ -0,0 +1,2 @@ +These files are manual copies of a pem cert from the /etc/ssl/certs/ directory. +They serve only as dummy certificate test files. diff --git a/test/core/security/etc/bundle.pem b/test/core/security/etc/bundle.pem new file mode 100644 index 00000000000..07d7672f830 --- /dev/null +++ b/test/core/security/etc/bundle.pem @@ -0,0 +1,63 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert1.pem b/test/core/security/etc/test_roots/cert1.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert1.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert2.pem b/test/core/security/etc/test_roots/cert2.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert2.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert3.pem b/test/core/security/etc/test_roots/cert3.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert3.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/linux_system_roots_test.cc b/test/core/security/linux_system_roots_test.cc new file mode 100644 index 00000000000..fce9c8dcc5d --- /dev/null +++ b/test/core/security/linux_system_roots_test.cc @@ -0,0 +1,104 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#ifdef GPR_LINUX +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/tmpfile.h" +#include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/security/context/security_context.h" +#include "src/core/lib/security/security_connector/load_system_roots.h" +#include "src/core/lib/security/security_connector/load_system_roots_linux.h" +#include "src/core/lib/security/security_connector/security_connector.h" +#include "src/core/lib/slice/slice_string_helpers.h" +#include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" +#include "test/core/util/test_config.h" + +#include "gtest/gtest.h" + +#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#endif + +namespace grpc { +namespace { + +TEST(AbsoluteFilePathTest, ConcatenatesCorrectly) { + const char* directory = "nonexistent/test/directory"; + const char* filename = "doesnotexist.txt"; + char result_path[MAXPATHLEN]; + grpc_core::GetAbsoluteFilePath(directory, filename, result_path); + EXPECT_STREQ(result_path, "nonexistent/test/directory/doesnotexist.txt"); +} + +TEST(CreateRootCertsBundleTest, ReturnsEmpty) { + // Test that CreateRootCertsBundle returns an empty slice for null or + // nonexistent cert directories. + grpc_slice result_slice = grpc_core::CreateRootCertsBundle(nullptr); + EXPECT_TRUE(GRPC_SLICE_IS_EMPTY(result_slice)); + grpc_slice_unref(result_slice); + result_slice = grpc_core::CreateRootCertsBundle("does/not/exist"); + EXPECT_TRUE(GRPC_SLICE_IS_EMPTY(result_slice)); + grpc_slice_unref(result_slice); +} + +TEST(CreateRootCertsBundleTest, BundlesCorrectly) { + gpr_setenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR, "true"); + // Test that CreateRootCertsBundle returns a correct slice. + grpc_slice roots_bundle = grpc_empty_slice(); + GRPC_LOG_IF_ERROR( + "load_file", + grpc_load_file("test/core/security/etc/bundle.pem", 1, &roots_bundle)); + // result_slice should have the same content as roots_bundle. + grpc_slice result_slice = + grpc_core::CreateRootCertsBundle("test/core/security/etc/test_roots"); + char* result_str = grpc_slice_to_c_string(result_slice); + char* bundle_str = grpc_slice_to_c_string(roots_bundle); + EXPECT_STREQ(result_str, bundle_str); + // Clean up. + unsetenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + gpr_free(result_str); + gpr_free(bundle_str); + grpc_slice_unref(roots_bundle); + grpc_slice_unref(result_slice); +} + +} // namespace +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +#else +int main() { + printf("*** WARNING: this test is only supported on Linux systems ***\n"); + return 0; +} +#endif // GPR_LINUX diff --git a/test/core/security/security_connector_test.cc b/test/core/security/security_connector_test.cc index e4c3ace6b4a..82d77eef8bd 100644 --- a/test/core/security/security_connector_test.cc +++ b/test/core/security/security_connector_test.cc @@ -363,7 +363,7 @@ static void test_ipv6_address_san(void) { namespace grpc_core { namespace { -class TestDefafaultSllRootStore : public DefaultSslRootStore { +class TestDefaultSslRootStore : public DefaultSslRootStore { public: static grpc_slice ComputePemRootCertsForTesting() { return ComputePemRootCerts(); @@ -389,7 +389,7 @@ static void test_default_ssl_roots(void) { gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); grpc_set_ssl_roots_override_callback(override_roots_success); grpc_slice roots = - grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); char* roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); @@ -398,7 +398,7 @@ static void test_default_ssl_roots(void) { /* Now let's set the env var: We should get the contents pointed value instead. */ gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_env_var_file_path); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_env_var) == 0); @@ -407,7 +407,7 @@ static void test_default_ssl_roots(void) { /* Now reset the env var. We should fall back to the value overridden using the api. */ gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); @@ -416,10 +416,10 @@ static void test_default_ssl_roots(void) { /* Now setup a permanent failure for the overridden roots and we should get an empty slice. */ grpc_set_ssl_roots_override_callback(override_roots_permanent_failure); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); GPR_ASSERT(GRPC_SLICE_IS_EMPTY(roots)); const tsi_ssl_root_certs_store* root_store = - grpc_core::TestDefafaultSllRootStore::GetRootStore(); + grpc_core::TestDefaultSslRootStore::GetRootStore(); GPR_ASSERT(root_store == nullptr); /* Cleanup. */ diff --git a/test/core/surface/completion_queue_test.cc b/test/core/surface/completion_queue_test.cc index 68129146cc7..b889fd0fc66 100644 --- a/test/core/surface/completion_queue_test.cc +++ b/test/core/surface/completion_queue_test.cc @@ -22,6 +22,7 @@ #include #include #include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/memory.h" #include "src/core/lib/iomgr/iomgr.h" #include "test/core/util/test_config.h" @@ -41,11 +42,18 @@ static void shutdown_and_destroy(grpc_completion_queue* cc) { case GRPC_CQ_NEXT: { ev = grpc_completion_queue_next(cc, gpr_inf_past(GPR_CLOCK_REALTIME), nullptr); + GPR_ASSERT(ev.type == GRPC_QUEUE_SHUTDOWN); break; } case GRPC_CQ_PLUCK: { ev = grpc_completion_queue_pluck( cc, create_test_tag(), gpr_inf_past(GPR_CLOCK_REALTIME), nullptr); + GPR_ASSERT(ev.type == GRPC_QUEUE_SHUTDOWN); + break; + } + case GRPC_CQ_CALLBACK: { + // Nothing to do here. The shutdown callback will be invoked when + // possible. break; } default: { @@ -54,7 +62,6 @@ static void shutdown_and_destroy(grpc_completion_queue* cc) { } } - GPR_ASSERT(ev.type == GRPC_QUEUE_SHUTDOWN); grpc_completion_queue_destroy(cc); } @@ -350,6 +357,76 @@ static void test_pluck_after_shutdown(void) { } } +static void test_callback(void) { + grpc_completion_queue* cc; + void* tags[128]; + grpc_cq_completion completions[GPR_ARRAY_SIZE(tags)]; + grpc_cq_polling_type polling_types[] = { + GRPC_CQ_DEFAULT_POLLING, GRPC_CQ_NON_LISTENING, GRPC_CQ_NON_POLLING}; + grpc_completion_queue_attributes attr; + unsigned i; + + LOG_TEST("test_callback"); + + bool got_shutdown = false; + class ShutdownCallback : public grpc_core::CQCallbackInterface { + public: + ShutdownCallback(bool* done) : done_(done) {} + ~ShutdownCallback() {} + void Run(bool ok) override { *done_ = ok; } + + private: + bool* done_; + }; + ShutdownCallback shutdown_cb(&got_shutdown); + + attr.version = 2; + attr.cq_completion_type = GRPC_CQ_CALLBACK; + attr.cq_shutdown_cb = &shutdown_cb; + + for (size_t pidx = 0; pidx < GPR_ARRAY_SIZE(polling_types); pidx++) { + grpc_core::ExecCtx exec_ctx; // reset exec_ctx + attr.cq_polling_type = polling_types[pidx]; + cc = grpc_completion_queue_create( + grpc_completion_queue_factory_lookup(&attr), &attr, nullptr); + + int counter = 0; + class TagCallback : public grpc_core::CQCallbackInterface { + public: + TagCallback(int* counter, int tag) : counter_(counter), tag_(tag) {} + ~TagCallback() {} + void Run(bool ok) override { + GPR_ASSERT(ok); + *counter_ += tag_; + grpc_core::Delete(this); + }; + + private: + int* counter_; + int tag_; + }; + + int sumtags = 0; + for (i = 0; i < GPR_ARRAY_SIZE(tags); i++) { + tags[i] = static_cast(grpc_core::New(&counter, i)); + sumtags += i; + } + + for (i = 0; i < GPR_ARRAY_SIZE(tags); i++) { + GPR_ASSERT(grpc_cq_begin_op(cc, tags[i])); + grpc_cq_end_op(cc, tags[i], GRPC_ERROR_NONE, do_nothing_end_completion, + nullptr, &completions[i]); + } + + GPR_ASSERT(sumtags == counter); + + shutdown_and_destroy(cc); + + GPR_ASSERT(got_shutdown); + got_shutdown = false; + } +} + struct thread_state { grpc_completion_queue* cc; void* tag; @@ -368,6 +445,7 @@ int main(int argc, char** argv) { test_pluck_after_shutdown(); test_cq_tls_cache_full(); test_cq_tls_cache_empty(); + test_callback(); grpc_shutdown(); return 0; } diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 9f4ad2b4d75..42dc95b9bb6 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -82,6 +82,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_completion_queue_factory_lookup); printf("%lx", (unsigned long) grpc_completion_queue_create_for_next); printf("%lx", (unsigned long) grpc_completion_queue_create_for_pluck); + printf("%lx", (unsigned long) grpc_completion_queue_create_for_callback); printf("%lx", (unsigned long) grpc_completion_queue_create); printf("%lx", (unsigned long) grpc_completion_queue_next); printf("%lx", (unsigned long) grpc_completion_queue_pluck); @@ -104,6 +105,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_census_call_get_context); printf("%lx", (unsigned long) grpc_channel_get_target); printf("%lx", (unsigned long) grpc_channel_get_info); + printf("%lx", (unsigned long) grpc_channel_reset_connect_backoff); printf("%lx", (unsigned long) grpc_insecure_channel_create); printf("%lx", (unsigned long) grpc_lame_client_channel_create); printf("%lx", (unsigned long) grpc_channel_destroy); @@ -130,6 +132,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_resource_quota_ref); printf("%lx", (unsigned long) grpc_resource_quota_unref); printf("%lx", (unsigned long) grpc_resource_quota_resize); + printf("%lx", (unsigned long) grpc_resource_quota_set_max_threads); printf("%lx", (unsigned long) grpc_resource_quota_arg_vtable); printf("%lx", (unsigned long) grpc_channelz_get_top_channels); printf("%lx", (unsigned long) grpc_channelz_get_channel); diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 3d31c9d8100..c9246f08068 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -1709,7 +1709,7 @@ TEST_P(AsyncEnd2endServerTryCancelTest, ServerBidiStreamingTryCancelAfter) { } std::vector CreateTestScenarios(bool test_secure, - int test_big_limit) { + bool test_message_size_limit) { std::vector scenarios; std::vector credentials_types; std::vector messages; @@ -1731,13 +1731,18 @@ std::vector CreateTestScenarios(bool test_secure, GPR_ASSERT(!credentials_types.empty()); messages.push_back("Hello"); - for (int sz = 1; sz <= test_big_limit; sz *= 32) { - grpc::string big_msg; - for (int i = 0; i < sz * 1024; i++) { - char c = 'a' + (i % 26); - big_msg += c; + if (test_message_size_limit) { + for (size_t k = 1; k < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH / 1024; + k *= 32) { + grpc::string big_msg; + for (size_t i = 0; i < k * 1024; ++i) { + char c = 'a' + (i % 26); + big_msg += c; + } + messages.push_back(big_msg); } - messages.push_back(big_msg); + messages.push_back( + grpc::string(GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH - 10, 'a')); } // TODO (sreek) Renable tests with health check service after the issue @@ -1758,10 +1763,10 @@ std::vector CreateTestScenarios(bool test_secure, } INSTANTIATE_TEST_CASE_P(AsyncEnd2end, AsyncEnd2endTest, - ::testing::ValuesIn(CreateTestScenarios(true, 1024))); + ::testing::ValuesIn(CreateTestScenarios(true, true))); INSTANTIATE_TEST_CASE_P(AsyncEnd2endServerTryCancel, AsyncEnd2endServerTryCancelTest, - ::testing::ValuesIn(CreateTestScenarios(false, 0))); + ::testing::ValuesIn(CreateTestScenarios(false, false))); } // namespace } // namespace testing diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index 8896fc6cae7..68219c16dcb 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -279,13 +279,29 @@ class ClientLbEnd2endTest : public ::testing::Test { void WaitForServer( const std::unique_ptr& stub, - size_t server_idx, const grpc_core::DebugLocation& location) { + size_t server_idx, const grpc_core::DebugLocation& location, + bool ignore_failure = false) { do { - CheckRpcSendOk(stub, location); + if (ignore_failure) { + SendRpc(stub); + } else { + CheckRpcSendOk(stub, location); + } } while (servers_[server_idx]->service_.request_count() == 0); ResetCounters(); } + bool WaitForChannelNotReady(Channel* channel, int timeout_seconds = 5) { + const gpr_timespec deadline = + grpc_timeout_seconds_to_deadline(timeout_seconds); + grpc_connectivity_state state; + while ((state = channel->GetState(false /* try_to_connect */)) == + GRPC_CHANNEL_READY) { + if (!channel->WaitForStateChange(state, deadline)) return false; + } + return true; + } + bool SeenAllServers() { for (const auto& server : servers_) { if (server->service_.request_count() == 0) return false; @@ -348,6 +364,23 @@ TEST_F(ClientLbEnd2endTest, PickFirst) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +TEST_F(ClientLbEnd2endTest, PickFirstProcessPending) { + StartServers(1); // Single server + auto channel = BuildChannel(""); // test that pick first is the default. + auto stub = BuildStub(channel); + SetNextResolution({servers_[0]->port_}); + WaitForServer(stub, 0, DEBUG_LOCATION); + // Create a new channel and its corresponding PF LB policy, which will pick + // the subchannels in READY state from the previous RPC against the same + // target (even if it happened over a different channel, because subchannels + // are globally reused). Progress should happen without any transition from + // this READY state. + auto second_channel = BuildChannel(""); + auto second_stub = BuildStub(second_channel); + SetNextResolution({servers_[0]->port_}); + CheckRpcSendOk(second_stub, DEBUG_LOCATION); +} + TEST_F(ClientLbEnd2endTest, PickFirstBackOffInitialReconnect) { ChannelArguments args; constexpr int kInitialBackOffMs = 100; @@ -403,6 +436,36 @@ TEST_F(ClientLbEnd2endTest, PickFirstBackOffMinReconnect) { gpr_atm_rel_store(&g_connection_delay_ms, 0); } +TEST_F(ClientLbEnd2endTest, PickFirstResetConnectionBackoff) { + ChannelArguments args; + constexpr int kInitialBackOffMs = 1000; + args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); + const std::vector ports = {grpc_pick_unused_port_or_die()}; + auto channel = BuildChannel("pick_first", args); + auto stub = BuildStub(channel); + SetNextResolution(ports); + // The channel won't become connected (there's no server). + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Bring up a server on the chosen port. + StartServers(1, ports); + const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC); + // Wait for connect, but not long enough. This proves that we're + // being throttled by initial backoff. + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Reset connection backoff. + experimental::ChannelResetConnectionBackoff(channel.get()); + // Wait for connect. Should happen ~immediately. + EXPECT_TRUE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC); + const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0)); + gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited_ms); + // We should have waited less than kInitialBackOffMs. + EXPECT_LT(waited_ms, kInitialBackOffMs); +} + TEST_F(ClientLbEnd2endTest, PickFirstUpdates) { // Start servers and send one RPC per server. const int kNumServers = 3; @@ -507,6 +570,81 @@ TEST_F(ClientLbEnd2endTest, PickFirstManyUpdates) { EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); } +TEST_F(ClientLbEnd2endTest, PickFirstReresolutionNoSelected) { + // Prepare the ports for up servers and down servers. + const int kNumServers = 3; + const int kNumAliveServers = 1; + StartServers(kNumAliveServers); + std::vector alive_ports, dead_ports; + for (size_t i = 0; i < kNumServers; ++i) { + if (i < kNumAliveServers) { + alive_ports.emplace_back(servers_[i]->port_); + } else { + dead_ports.emplace_back(grpc_pick_unused_port_or_die()); + } + } + auto channel = BuildChannel("pick_first"); + auto stub = BuildStub(channel); + // The initial resolution only contains dead ports. There won't be any + // selected subchannel. Re-resolution will return the same result. + SetNextResolution(dead_ports); + gpr_log(GPR_INFO, "****** INITIAL RESOLUTION SET *******"); + for (size_t i = 0; i < 10; ++i) CheckRpcSendFailure(stub); + // Set a re-resolution result that contains reachable ports, so that the + // pick_first LB policy can recover soon. + SetNextResolutionUponError(alive_ports); + gpr_log(GPR_INFO, "****** RE-RESOLUTION SET *******"); + WaitForServer(stub, 0, DEBUG_LOCATION, true /* ignore_failure */); + CheckRpcSendOk(stub, DEBUG_LOCATION); + EXPECT_EQ(servers_[0]->service_.request_count(), 1); + // Check LB policy name for the channel. + EXPECT_EQ("pick_first", channel->GetLoadBalancingPolicyName()); +} + +TEST_F(ClientLbEnd2endTest, PickFirstCheckStateBeforeStartWatch) { + std::vector ports = {grpc_pick_unused_port_or_die()}; + StartServers(1, ports); + auto channel_1 = BuildChannel("pick_first"); + auto stub_1 = BuildStub(channel_1); + SetNextResolution(ports); + gpr_log(GPR_INFO, "****** RESOLUTION SET FOR CHANNEL 1 *******"); + WaitForServer(stub_1, 0, DEBUG_LOCATION); + gpr_log(GPR_INFO, "****** CHANNEL 1 CONNECTED *******"); + servers_[0]->Shutdown(); + // Channel 1 will receive a re-resolution containing the same server. It will + // create a new subchannel and hold a ref to it. + servers_.clear(); + StartServers(1, ports); + gpr_log(GPR_INFO, "****** SERVER RESTARTED *******"); + auto channel_2 = BuildChannel("pick_first"); + auto stub_2 = BuildStub(channel_2); + // TODO(juanlishen): This resolution result will only be visible to channel 2 + // since the response generator is only associated with channel 2 now. We + // should change the response generator to be able to deliver updates to + // multiple channels at once. + SetNextResolution(ports); + gpr_log(GPR_INFO, "****** RESOLUTION SET FOR CHANNEL 2 *******"); + WaitForServer(stub_2, 0, DEBUG_LOCATION, true); + gpr_log(GPR_INFO, "****** CHANNEL 2 CONNECTED *******"); + servers_[0]->Shutdown(); + // Wait until the disconnection has triggered the connectivity notification. + // Otherwise, the subchannel may be picked for next call but will fail soon. + EXPECT_TRUE(WaitForChannelNotReady(channel_2.get())); + // Channel 2 will also receive a re-resolution containing the same server. + // Both channels will ref the same subchannel that failed. + servers_.clear(); + StartServers(1, ports); + gpr_log(GPR_INFO, "****** SERVER RESTARTED AGAIN *******"); + gpr_log(GPR_INFO, "****** CHANNEL 2 STARTING A CALL *******"); + // The first call after the server restart will succeed. + CheckRpcSendOk(stub_2, DEBUG_LOCATION); + gpr_log(GPR_INFO, "****** CHANNEL 2 FINISHED A CALL *******"); + // Check LB policy name for the channel. + EXPECT_EQ("pick_first", channel_1->GetLoadBalancingPolicyName()); + // Check LB policy name for the channel. + EXPECT_EQ("pick_first", channel_2->GetLoadBalancingPolicyName()); +} + TEST_F(ClientLbEnd2endTest, RoundRobin) { // Start servers and send one RPC per server. const int kNumServers = 3; diff --git a/test/cpp/end2end/grpclb_end2end_test.cc b/test/cpp/end2end/grpclb_end2end_test.cc index 28f9ae6f40a..b69b861fcf4 100644 --- a/test/cpp/end2end/grpclb_end2end_test.cc +++ b/test/cpp/end2end/grpclb_end2end_test.cc @@ -734,6 +734,25 @@ TEST_F(SingleBalancerTest, InitiallyEmptyServerlist) { EXPECT_EQ(2U, balancer_servers_[0].service_->response_count()); } +TEST_F(SingleBalancerTest, AllServersUnreachableFailFast) { + SetNextResolutionAllBalancers(); + const size_t kNumUnreachableServers = 5; + std::vector ports; + for (size_t i = 0; i < kNumUnreachableServers; ++i) { + ports.push_back(grpc_pick_unused_port_or_die()); + } + ScheduleResponseForBalancer( + 0, BalancerServiceImpl::BuildResponseForBackends(ports, {}), 0); + const Status status = SendRpc(); + // The error shouldn't be DEADLINE_EXCEEDED. + EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code()); + balancers_[0]->NotifyDoneWithServerlists(); + // The balancer got a single request. + EXPECT_EQ(1U, balancer_servers_[0].service_->request_count()); + // and sent a single response. + EXPECT_EQ(1U, balancer_servers_[0].service_->response_count()); +} + TEST_F(SingleBalancerTest, Fallback) { SetNextResolutionAllBalancers(); const int kFallbackTimeoutMs = 200 * grpc_test_slowdown_factor(); diff --git a/test/cpp/end2end/thread_stress_test.cc b/test/cpp/end2end/thread_stress_test.cc index ccf8400a872..1a5ed28a2cc 100644 --- a/test/cpp/end2end/thread_stress_test.cc +++ b/test/cpp/end2end/thread_stress_test.cc @@ -16,6 +16,7 @@ * */ +#include #include #include @@ -24,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -51,63 +53,13 @@ namespace testing { class TestServiceImpl : public ::grpc::testing::EchoTestService::Service { public: - TestServiceImpl() : signal_client_(false) {} + TestServiceImpl() {} Status Echo(ServerContext* context, const EchoRequest* request, EchoResponse* response) override { response->set_message(request->message()); return Status::OK; } - - // Unimplemented is left unimplemented to test the returned error. - - Status RequestStream(ServerContext* context, - ServerReader* reader, - EchoResponse* response) override { - EchoRequest request; - response->set_message(""); - while (reader->Read(&request)) { - response->mutable_message()->append(request.message()); - } - return Status::OK; - } - - // Return 3 messages. - // TODO(yangg) make it generic by adding a parameter into EchoRequest - Status ResponseStream(ServerContext* context, const EchoRequest* request, - ServerWriter* writer) override { - EchoResponse response; - response.set_message(request->message() + "0"); - writer->Write(response); - response.set_message(request->message() + "1"); - writer->Write(response); - response.set_message(request->message() + "2"); - writer->Write(response); - - return Status::OK; - } - - Status BidiStream( - ServerContext* context, - ServerReaderWriter* stream) override { - EchoRequest request; - EchoResponse response; - while (stream->Read(&request)) { - gpr_log(GPR_INFO, "recv msg %s", request.message().c_str()); - response.set_message(request.message()); - stream->Write(response); - } - return Status::OK; - } - - bool signal_client() { - std::unique_lock lock(mu_); - return signal_client_; - } - - private: - bool signal_client_; - std::mutex mu_; }; template @@ -118,6 +70,7 @@ class CommonStressTest { virtual void SetUp() = 0; virtual void TearDown() = 0; virtual void ResetStub() = 0; + virtual bool AllowExhaustion() = 0; grpc::testing::EchoTestService::Stub* GetStub() { return stub_.get(); } protected: @@ -146,6 +99,7 @@ class CommonStressTestInsecure : public CommonStressTest { CreateChannel(server_address_.str(), InsecureChannelCredentials()); this->stub_ = grpc::testing::EchoTestService::NewStub(channel); } + bool AllowExhaustion() override { return false; } protected: void SetUpStart(ServerBuilder* builder, Service* service) override { @@ -161,7 +115,7 @@ class CommonStressTestInsecure : public CommonStressTest { std::ostringstream server_address_; }; -template +template class CommonStressTestInproc : public CommonStressTest { public: void ResetStub() override { @@ -169,6 +123,7 @@ class CommonStressTestInproc : public CommonStressTest { std::shared_ptr channel = this->server_->InProcessChannel(args); this->stub_ = grpc::testing::EchoTestService::NewStub(channel); } + bool AllowExhaustion() override { return allow_resource_exhaustion; } protected: void SetUpStart(ServerBuilder* builder, Service* service) override { @@ -193,6 +148,26 @@ class CommonStressTestSyncServer : public BaseClass { TestServiceImpl service_; }; +template +class CommonStressTestSyncServerLowThreadCount : public BaseClass { + public: + void SetUp() override { + ServerBuilder builder; + ResourceQuota quota; + this->SetUpStart(&builder, &service_); + quota.SetMaxThreads(4); + builder.SetResourceQuota(quota); + this->SetUpEnd(&builder); + } + void TearDown() override { + this->TearDownStart(); + this->TearDownEnd(); + } + + private: + TestServiceImpl service_; +}; + template class CommonStressTestAsyncServer : public BaseClass { public: @@ -293,7 +268,8 @@ class End2endTest : public ::testing::Test { Common common_; }; -static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) { +static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs, + bool allow_exhaustion, gpr_atm* errors) { EchoRequest request; EchoResponse response; request.set_message("Hello"); @@ -301,34 +277,53 @@ static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) { for (int i = 0; i < num_rpcs; ++i) { ClientContext context; Status s = stub->Echo(&context, request, &response); - EXPECT_EQ(response.message(), request.message()); + EXPECT_TRUE(s.ok() || (allow_exhaustion && + s.error_code() == StatusCode::RESOURCE_EXHAUSTED)); if (!s.ok()) { - gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(), - s.error_message().c_str()); + if (!(allow_exhaustion && + s.error_code() == StatusCode::RESOURCE_EXHAUSTED)) { + gpr_log(GPR_ERROR, "RPC error: %d: %s", s.error_code(), + s.error_message().c_str()); + } + gpr_atm_no_barrier_fetch_add(errors, static_cast(1)); + } else { + EXPECT_EQ(response.message(), request.message()); } - ASSERT_TRUE(s.ok()); } } typedef ::testing::Types< CommonStressTestSyncServer>, - CommonStressTestSyncServer>, + CommonStressTestSyncServer>, + CommonStressTestSyncServerLowThreadCount< + CommonStressTestInproc>, CommonStressTestAsyncServer< CommonStressTestInsecure>, - CommonStressTestAsyncServer< - CommonStressTestInproc>> + CommonStressTestAsyncServer>> CommonTypes; TYPED_TEST_CASE(End2endTest, CommonTypes); TYPED_TEST(End2endTest, ThreadStress) { this->common_.ResetStub(); std::vector threads; + gpr_atm errors; + gpr_atm_rel_store(&errors, static_cast(0)); threads.reserve(kNumThreads); for (int i = 0; i < kNumThreads; ++i) { - threads.emplace_back(SendRpc, this->common_.GetStub(), kNumRpcs); + threads.emplace_back(SendRpc, this->common_.GetStub(), kNumRpcs, + this->common_.AllowExhaustion(), &errors); } for (int i = 0; i < kNumThreads; ++i) { threads[i].join(); } + uint64_t error_cnt = static_cast(gpr_atm_no_barrier_load(&errors)); + if (error_cnt != 0) { + gpr_log(GPR_INFO, "RPC error count: %" PRIu64, error_cnt); + } + // If this test allows resource exhaustion, expect that it actually sees some + if (this->common_.AllowExhaustion()) { + EXPECT_GT(error_cnt, static_cast(0)); + } } template diff --git a/test/cpp/interop/client_helper.cc b/test/cpp/interop/client_helper.cc index 29b5a1ed6c2..fb7b7bb7d03 100644 --- a/test/cpp/interop/client_helper.cc +++ b/test/cpp/interop/client_helper.cc @@ -88,20 +88,20 @@ std::shared_ptr CreateChannelForTestCase( std::shared_ptr creds; if (test_case == "compute_engine_creds") { - GPR_ASSERT(FLAGS_use_tls); - creds = GoogleComputeEngineCredentials(); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : GoogleComputeEngineCredentials(); } else if (test_case == "jwt_token_creds") { - GPR_ASSERT(FLAGS_use_tls); grpc::string json_key = GetServiceAccountJsonKey(); std::chrono::seconds token_lifetime = std::chrono::hours(1); - creds = - ServiceAccountJWTAccessCredentials(json_key, token_lifetime.count()); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : ServiceAccountJWTAccessCredentials(json_key, + token_lifetime.count()); } else if (test_case == "oauth2_auth_token") { - grpc::string raw_token = GetOauth2AccessToken(); - creds = AccessTokenCredentials(raw_token); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : AccessTokenCredentials(GetOauth2AccessToken()); } if (FLAGS_custom_credentials_type.empty()) { transport_security security_type = diff --git a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc index da095c3e68b..85767c8758f 100644 --- a/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc +++ b/test/cpp/microbenchmarks/bm_cq_multiple_threads.cc @@ -34,15 +34,15 @@ struct grpc_pollset { gpr_mu mu; }; +static gpr_mu g_mu; +static gpr_cv g_cv; +static int g_threads_active; +static bool g_active; + namespace grpc { namespace testing { - -auto& force_library_initialization = Library::get(); - -static void* g_tag = (void*)static_cast(10); // Some random number static grpc_completion_queue* g_cq; static grpc_event_engine_vtable g_vtable; -static const grpc_event_engine_vtable* g_old_vtable; static void pollset_shutdown(grpc_pollset* ps, grpc_closure* closure) { GRPC_CLOSURE_SCHED(closure, GRPC_ERROR_NONE); @@ -74,16 +74,18 @@ static grpc_error* pollset_work(grpc_pollset* ps, grpc_pollset_worker** worker, } gpr_mu_unlock(&ps->mu); - GPR_ASSERT(grpc_cq_begin_op(g_cq, g_tag)); + + void* tag = (void*)static_cast(10); // Some random number + GPR_ASSERT(grpc_cq_begin_op(g_cq, tag)); grpc_cq_end_op( - g_cq, g_tag, GRPC_ERROR_NONE, cq_done_cb, nullptr, + g_cq, tag, GRPC_ERROR_NONE, cq_done_cb, nullptr, static_cast(gpr_malloc(sizeof(grpc_cq_completion)))); grpc_core::ExecCtx::Get()->Flush(); gpr_mu_lock(&ps->mu); return GRPC_ERROR_NONE; } -static void init_engine_vtable() { +static const grpc_event_engine_vtable* init_engine_vtable(bool) { memset(&g_vtable, 0, sizeof(g_vtable)); g_vtable.pollset_size = sizeof(grpc_pollset); @@ -92,17 +94,23 @@ static void init_engine_vtable() { g_vtable.pollset_destroy = pollset_destroy; g_vtable.pollset_work = pollset_work; g_vtable.pollset_kick = pollset_kick; + g_vtable.shutdown_engine = [] {}; + + return &g_vtable; } static void setup() { - grpc_init(); + // This test should only ever be run with a non or any polling engine + // Override the polling engine for the non-polling engine + // and add a custom polling engine + grpc_register_event_engine_factory("none", init_engine_vtable, false); + grpc_register_event_engine_factory("bm_cq_multiple_threads", + init_engine_vtable, true); - /* Override the event engine with our test event engine (g_vtable); but before - * that, save the current event engine in g_old_vtable. We will have to set - * g_old_vtable back before calling grpc_shutdown() */ - init_engine_vtable(); - g_old_vtable = grpc_get_event_engine_test_only(); - grpc_set_event_engine_test_only(&g_vtable); + grpc_init(); + GPR_ASSERT(strcmp(grpc_get_poll_strategy_name(), "none") == 0 || + strcmp(grpc_get_poll_strategy_name(), "bm_cq_multiple_threads") == + 0); g_cq = grpc_completion_queue_create_for_next(nullptr); } @@ -118,9 +126,6 @@ static void teardown() { } grpc_completion_queue_destroy(g_cq); - - /* Restore the old event engine before calling grpc_shutdown */ - grpc_set_event_engine_test_only(g_old_vtable); grpc_shutdown(); } @@ -137,14 +142,33 @@ static void teardown() { code (i.e the code between two successive calls of state.KeepRunning()) if state.KeepRunning() returns false. So it is safe to do the teardown in one of the threads after state.keepRunning() returns false. + + However, our use requires synchronization because we do additional work at + each thread that requires specific ordering (TrackCounters must be constructed + after grpc_init because it needs the number of cores, initialized by grpc, + and its Finish call must take place before grpc_shutdown so that it can use + grpc_stats). */ static void BM_Cq_Throughput(benchmark::State& state) { - TrackCounters track_counters; gpr_timespec deadline = gpr_inf_future(GPR_CLOCK_MONOTONIC); + auto thd_idx = state.thread_index; - if (state.thread_index == 0) { + gpr_mu_lock(&g_mu); + g_threads_active++; + if (thd_idx == 0) { setup(); + g_active = true; + gpr_cv_broadcast(&g_cv); + } else { + while (!g_active) { + gpr_cv_wait(&g_cv, &g_mu, deadline); + } } + gpr_mu_unlock(&g_mu); + + // Use a TrackCounters object to monitor the gRPC performance statistics + // (optionally including low-level counters) before and after the test + TrackCounters track_counters; while (state.KeepRunning()) { GPR_ASSERT(grpc_completion_queue_next(g_cq, deadline, nullptr).type == @@ -152,12 +176,23 @@ static void BM_Cq_Throughput(benchmark::State& state) { } state.SetItemsProcessed(state.iterations()); + track_counters.Finish(state); - if (state.thread_index == 0) { - teardown(); + gpr_mu_lock(&g_mu); + g_threads_active--; + if (g_threads_active == 0) { + gpr_cv_broadcast(&g_cv); + } else { + while (g_threads_active > 0) { + gpr_cv_wait(&g_cv, &g_mu, deadline); + } } + gpr_mu_unlock(&g_mu); - track_counters.Finish(state); + if (thd_idx == 0) { + teardown(); + g_active = false; + } } BENCHMARK(BM_Cq_Throughput)->ThreadRange(1, 16)->UseRealTime(); @@ -172,6 +207,8 @@ void RunTheBenchmarksNamespaced() { RunSpecifiedBenchmarks(); } } // namespace benchmark int main(int argc, char** argv) { + gpr_mu_init(&g_mu); + gpr_cv_init(&g_cv); ::benchmark::Initialize(&argc, argv); ::grpc::testing::InitTest(&argc, &argv, false); benchmark::RunTheBenchmarksNamespaced(); diff --git a/test/cpp/naming/address_sorting_test.cc b/test/cpp/naming/address_sorting_test.cc index a92e9e3b3e3..04c300876cc 100644 --- a/test/cpp/naming/address_sorting_test.cc +++ b/test/cpp/naming/address_sorting_test.cc @@ -24,10 +24,8 @@ #include #include -#include #include #include -#include #include #include @@ -51,6 +49,11 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#ifndef GPR_WINDOWS +#include +#include +#endif + namespace { struct TestAddress { @@ -190,10 +193,18 @@ void VerifyLbAddrOutputs(grpc_lb_addresses* lb_addrs, grpc_lb_addresses_destroy(lb_addrs); } -} // namespace +/* We need to run each test case inside of its own + * isolated grpc_init/grpc_shutdown pair, so that + * the "address sorting source addr factory" can be + * restored to its default for each test case. */ +class AddressSortingTest : public ::testing::Test { + protected: + void SetUp() override { grpc_init(); } + void TearDown() override { grpc_shutdown(); } +}; /* Tests for rule 1 */ -TEST(AddressSortingTest, TestDepriotizesUnreachableAddresses) { +TEST_F(AddressSortingTest, TestDepriotizesUnreachableAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -212,7 +223,7 @@ TEST(AddressSortingTest, TestDepriotizesUnreachableAddresses) { }); } -TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { +TEST_F(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { bool ipv4_supported = true; bool ipv6_supported = false; OverrideAddressSortingSourceAddrFactory( @@ -231,7 +242,7 @@ TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv6) { }); } -TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { +TEST_F(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { bool ipv4_supported = false; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -253,7 +264,7 @@ TEST(AddressSortingTest, TestDepriotizesUnsupportedDomainIpv4) { /* Tests for rule 2 */ -TEST(AddressSortingTest, TestDepriotizesNonMatchingScope) { +TEST_F(AddressSortingTest, TestDepriotizesNonMatchingScope) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -277,7 +288,7 @@ TEST(AddressSortingTest, TestDepriotizesNonMatchingScope) { /* Tests for rule 5 */ -TEST(AddressSortingTest, TestUsesLabelFromDefaultTable) { +TEST_F(AddressSortingTest, TestUsesLabelFromDefaultTable) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -300,7 +311,7 @@ TEST(AddressSortingTest, TestUsesLabelFromDefaultTable) { /* Flip the input on the test above to reorder the sort function's * comparator's inputs. */ -TEST(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { +TEST_F(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -323,8 +334,8 @@ TEST(AddressSortingTest, TestUsesLabelFromDefaultTableInputFlipped) { /* Tests for rule 6 */ -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithAnIpv4Address) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithAnIpv4Address) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -348,8 +359,8 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithV4CompatAndLocalhostAddress) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithV4CompatAndLocalhostAddress) { bool ipv4_supported = true; bool ipv6_supported = true; // Handle unique observed behavior of inet_ntop(v4-compatible-address) on OS X. @@ -377,8 +388,8 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithCatchAllAndLocalhostAddress) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithCatchAllAndLocalhostAddress) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -403,8 +414,8 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWith2000PrefixedAddress) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWith2000PrefixedAddress) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -426,7 +437,7 @@ TEST(AddressSortingTest, }); } -TEST( +TEST_F( AddressSortingTest, TestUsesDestinationWithHigherPrecedenceWith2000PrefixedAddressEnsurePrefixMatchHasNoEffect) { bool ipv4_supported = true; @@ -448,8 +459,8 @@ TEST( }); } -TEST(AddressSortingTest, - TestUsesDestinationWithHigherPrecedenceWithLinkAndSiteLocalAddresses) { +TEST_F(AddressSortingTest, + TestUsesDestinationWithHigherPrecedenceWithLinkAndSiteLocalAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -469,19 +480,22 @@ TEST(AddressSortingTest, }); } -TEST( +TEST_F( AddressSortingTest, TestUsesDestinationWithHigherPrecedenceWithCatchAllAndAndV4MappedAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; + // Use embedded ipv4 addresses with leading 1's instead of zero's to be + // compatible with inet_ntop implementations that can display such + // addresses with leading zero's as e.g.: "::ffff:0:2", as on windows. OverrideAddressSortingSourceAddrFactory( ipv4_supported, ipv6_supported, { - {"[::ffff:0.0.0.2]:443", {"[::ffff:0.0.0.3]:0", AF_INET6}}, + {"[::ffff:1.1.1.2]:443", {"[::ffff:1.1.1.3]:0", AF_INET6}}, {"[1234::2]:443", {"[1234::3]:0", AF_INET6}}, }); grpc_lb_addresses* lb_addrs = BuildLbAddrInputs({ - {"[::ffff:0.0.0.2]:443", AF_INET6}, + {"[::ffff:1.1.1.2]:443", AF_INET6}, {"[1234::2]:443", AF_INET6}, }); grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); @@ -489,13 +503,13 @@ TEST( // ::ffff:0:2 should match the v4-mapped // precedence entry and be deprioritized. "[1234::2]:443", - "[::ffff:0.0.0.2]:443", + "[::ffff:1.1.1.2]:443", }); } /* Tests for rule 8 */ -TEST(AddressSortingTest, TestPrefersSmallerScope) { +TEST_F(AddressSortingTest, TestPrefersSmallerScope) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -520,7 +534,7 @@ TEST(AddressSortingTest, TestPrefersSmallerScope) { /* Tests for rule 9 */ -TEST(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { +TEST_F(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -543,8 +557,8 @@ TEST(AddressSortingTest, TestPrefersLongestMatchingSrcDstPrefix) { }); } -TEST(AddressSortingTest, - TestPrefersLongestMatchingSrcDstPrefixMatchesWholeAddress) { +TEST_F(AddressSortingTest, + TestPrefersLongestMatchingSrcDstPrefixMatchesWholeAddress) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -564,7 +578,7 @@ TEST(AddressSortingTest, }); } -TEST(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { +TEST_F(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -584,7 +598,7 @@ TEST(AddressSortingTest, TestPrefersLongestPrefixStressInnerBytePrefix) { }); } -TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { +TEST_F(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -604,7 +618,7 @@ TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersOnHighestBitOfByte) { }); } -TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { +TEST_F(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -628,7 +642,7 @@ TEST(AddressSortingTest, TestPrefersLongestPrefixDiffersByLastBit) { /* Tests for rule 10 */ -TEST(AddressSortingTest, TestStableSort) { +TEST_F(AddressSortingTest, TestStableSort) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -648,7 +662,7 @@ TEST(AddressSortingTest, TestStableSort) { }); } -TEST(AddressSortingTest, TestStableSortFiveElements) { +TEST_F(AddressSortingTest, TestStableSortFiveElements) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory( @@ -677,7 +691,7 @@ TEST(AddressSortingTest, TestStableSortFiveElements) { }); } -TEST(AddressSortingTest, TestStableSortNoSrcAddrsExist) { +TEST_F(AddressSortingTest, TestStableSortNoSrcAddrsExist) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory(ipv4_supported, ipv6_supported, {}); @@ -698,7 +712,7 @@ TEST(AddressSortingTest, TestStableSortNoSrcAddrsExist) { }); } -TEST(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { +TEST_F(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { bool ipv4_supported = true; bool ipv6_supported = true; OverrideAddressSortingSourceAddrFactory(ipv4_supported, ipv6_supported, {}); @@ -713,7 +727,7 @@ TEST(AddressSortingTest, TestStableSortNoSrcAddrsExistWithIpv4) { }); } -TEST(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { +TEST_F(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { bool ipv4_supported = true; bool ipv6_supported = true; // Handle unique observed behavior of inet_ntop(v4-compatible-address) on OS X. @@ -744,6 +758,78 @@ TEST(AddressSortingTest, TestStableSortV4CompatAndSiteLocalAddresses) { }); } +/* TestPrefersIpv6Loopback tests the actual "address probing" code + * for the current platform, without any mocks. + * This test relies on the assumption that the ipv6 loopback address is + * available in the hosts/containers that grpc C/C++ tests run on + * (whether ipv4 loopback is available or not, an available ipv6 + * loopback should be preferred). */ +TEST_F(AddressSortingTest, TestPrefersIpv6Loopback) { + grpc_lb_addresses* lb_addrs = BuildLbAddrInputs({ + {"[::1]:443", AF_INET6}, + {"127.0.0.1:443", AF_INET}, + }); + grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + VerifyLbAddrOutputs(lb_addrs, { + "[::1]:443", + "127.0.0.1:443", + }); +} + +/* Flip the order of the inputs above and expect the same output order + * (try to rule out influence of arbitrary qsort ordering) */ +TEST_F(AddressSortingTest, TestPrefersIpv6LoopbackInputsFlipped) { + grpc_lb_addresses* lb_addrs = BuildLbAddrInputs({ + {"127.0.0.1:443", AF_INET}, + {"[::1]:443", AF_INET6}, + }); + grpc_cares_wrapper_test_only_address_sorting_sort(lb_addrs); + VerifyLbAddrOutputs(lb_addrs, { + "[::1]:443", + "127.0.0.1:443", + }); +} + +/* Try to rule out false positives in the above two tests in which + * the sorter might think that neither ipv6 or ipv4 loopback is + * available, but ipv6 loopback is still preferred only due + * to precedance table lookups. */ +TEST_F(AddressSortingTest, TestSorterKnowsIpv6LoopbackIsAvailable) { + sockaddr_in6 ipv6_loopback; + memset(&ipv6_loopback, 0, sizeof(ipv6_loopback)); + ipv6_loopback.sin6_family = AF_INET6; + ((char*)&ipv6_loopback.sin6_addr)[15] = 1; + ipv6_loopback.sin6_port = htons(443); + // Set up the source and destination parameters of + // address_sorting_get_source_addr + address_sorting_address sort_input_dest; + memcpy(&sort_input_dest.addr, &ipv6_loopback, sizeof(ipv6_loopback)); + sort_input_dest.len = sizeof(ipv6_loopback); + address_sorting_address source_for_sort_input_dest; + memset(&source_for_sort_input_dest, 0, sizeof(source_for_sort_input_dest)); + // address_sorting_get_source_addr returns true if a source address was found + // for the destination address, otherwise false. + EXPECT_TRUE(address_sorting_get_source_addr_for_testing( + &sort_input_dest, &source_for_sort_input_dest)); + // Now also check that the source address was filled in correctly. + EXPECT_GT(source_for_sort_input_dest.len, 0u); + sockaddr_in6* source_addr_output = + (sockaddr_in6*)source_for_sort_input_dest.addr; + EXPECT_EQ(source_addr_output->sin6_family, AF_INET6); + char* buf = static_cast(gpr_zalloc(100)); + EXPECT_NE(inet_ntop(AF_INET6, &source_addr_output->sin6_addr, buf, 100), + nullptr) + << "inet_ntop failed. Errno: " + std::to_string(errno); + std::string source_addr_str(buf); + gpr_free(buf); + // This test + // assumes that the source address for any loopback destination is also the + // loopback address. + EXPECT_EQ(source_addr_str, "::1"); +} + +} // namespace + int main(int argc, char** argv) { char* resolver = gpr_getenv("GRPC_DNS_RESOLVER"); if (resolver == nullptr || strlen(resolver) == 0) { @@ -754,9 +840,7 @@ int main(int argc, char** argv) { gpr_free(resolver); grpc_test_init(argc, argv); ::testing::InitGoogleTest(&argc, argv); - grpc_init(); auto result = RUN_ALL_TESTS(); - grpc_shutdown(); // Test sequential and nested inits and shutdowns. grpc_init(); grpc_init(); diff --git a/test/cpp/naming/cancel_ares_query_test.cc b/test/cpp/naming/cancel_ares_query_test.cc index 0d59bf6fb60..dec7c171dc0 100644 --- a/test/cpp/naming/cancel_ares_query_test.cc +++ b/test/cpp/naming/cancel_ares_query_test.cc @@ -45,11 +45,14 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -// TODO: pull in different headers when enabling this -// test on windows. Also set BAD_SOCKET_RETURN_VAL -// to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif namespace { @@ -91,7 +94,13 @@ class FakeNonResponsiveDNSServer { abort(); } } - ~FakeNonResponsiveDNSServer() { close(socket_); } + ~FakeNonResponsiveDNSServer() { +#ifdef GPR_WINDOWS + closesocket(socket_); +#else + close(socket_); +#endif + } private: int socket_; @@ -193,6 +202,38 @@ TEST(CancelDuringAresQuery, TestCancelActiveDNSQuery) { TestCancelActiveDNSQuery(&args); } +#ifdef GPR_WINDOWS + +void MaybePollArbitraryPollsetTwice() { + grpc_pollset* pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size()); + gpr_mu* mu; + grpc_pollset_init(pollset, &mu); + grpc_pollset_worker* worker = nullptr; + // Make a zero timeout poll + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + // Make a second zero-timeout poll (in case the first one + // short-circuited by picking up a previous "kick") + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + grpc_pollset_destroy(pollset); + gpr_free(pollset); +} + +#else + +void MaybePollArbitraryPollsetTwice() {} + +#endif + TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { grpc_core::ExecCtx exec_ctx; ArgsStruct args; @@ -209,6 +250,12 @@ TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { // this test. This test only cares about what happens to fd's that c-ares // opens. TestCancelActiveDNSQuery(&args); + // This test relies on the assumption that cancelling a c-ares query + // will flush out all callbacks on the current exec ctx, which is true + // on posix platforms but not on Windows, because fd shutdown on Windows + // requires a trip through the polling loop to schedule the callback. + // So we need to do extra polling work on Windows to free things up. + MaybePollArbitraryPollsetTwice(); EXPECT_EQ(grpc_iomgr_count_objects_for_testing(), 0u); grpc_pollset_set_destroy(fake_other_pollset_set); } diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index baa6512f624..1c9d0676b89 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -68,7 +68,7 @@ def main(): 'gtest': False, 'run': False, 'src': ['test/cpp/naming/resolver_component_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util' + unsecure_build_config_suffix, 'grpc_test_util' + unsecure_build_config_suffix, @@ -110,7 +110,7 @@ def main(): 'gtest': True, 'run': True, 'src': ['test/cpp/naming/address_sorting_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util' + unsecure_build_config_suffix, 'grpc_test_util' + unsecure_build_config_suffix, @@ -129,7 +129,7 @@ def main(): 'gtest': True, 'run': True, 'src': ['test/cpp/naming/cancel_ares_query_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util', 'grpc_test_util', diff --git a/test/cpp/naming/manual_run_resolver_component_test.py b/test/cpp/naming/manual_run_resolver_component_test.py new file mode 100644 index 00000000000..fb2157741a0 --- /dev/null +++ b/test/cpp/naming/manual_run_resolver_component_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +# The c-ares test suite doesn't get ran regularly on Windows, but +# this script provides a way to run a lot of the tests manually. +_MSBUILD_CONFIG = os.environ['CONFIG'] +os.chdir(os.path.join('..', '..', os.getcwd())) +# This port is arbitrary, but it needs to be available. +_DNS_SERVER_PORT = 15353 + +subprocess.call([ + sys.executable, + 'test\\cpp\\naming\\resolver_component_tests_runner.py', + '--test_bin_path', 'cmake\\build\\%s\\resolver_component_test.exe' % _MSBUILD_CONFIG, + '--dns_server_bin_path', 'test\\cpp\\naming\\utils\\dns_server.py', + '--records_config_path', 'test\\cpp\\naming\\resolver_test_record_groups.yaml', + '--dns_server_port', str(_DNS_SERVER_PORT), + '--dns_resolver_bin_path', 'test\\cpp\\naming\\utils\\dns_resolver.py', + '--tcp_connect_bin_path', 'test\\cpp\\naming\\utils\\tcp_connect.py', +]) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 6ac548120ca..3dc6e7178cc 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -16,6 +16,8 @@ * */ +#include + #include #include #include @@ -55,8 +57,15 @@ // TODO: pull in different headers when enabling this // test on windows. Also set BAD_SOCKET_RETURN_VAL // to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif using grpc::SubProcess; using std::vector; @@ -241,6 +250,62 @@ void CheckLBPolicyResultLocked(grpc_channel_args* channel_args, } } +#ifdef GPR_WINDOWS +void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { + sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(dummy_port); + ((char*)&addr.sin6_addr)[15] = 1; + for (;;) { + if (gpr_event_get(done_ev)) { + return; + } + std::vector sockets; + for (size_t i = 0; i < 50; i++) { + SOCKET s = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, + WSA_FLAG_OVERLAPPED); + ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL) + << "Failed to create TCP ipv6 socket"; + gpr_log(GPR_DEBUG, "Opened socket: %d", s); + char val = 1; + ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) != + SOCKET_ERROR) + << "Failed to set socketopt reuseaddr. WSA error: " + + std::to_string(WSAGetLastError()); + ASSERT_TRUE(grpc_tcp_set_non_block(s) == GRPC_ERROR_NONE) + << "Failed to set socket non-blocking"; + ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) != SOCKET_ERROR) + << "Failed to bind socket " + std::to_string(s) + + " to [::1]:" + std::to_string(dummy_port) + + ". WSA error: " + std::to_string(WSAGetLastError()); + ASSERT_TRUE(listen(s, 1) != SOCKET_ERROR) + << "Failed to listen on socket " + std::to_string(s) + + ". WSA error: " + std::to_string(WSAGetLastError()); + sockets.push_back(s); + } + // Do a non-blocking accept followed by a close on all of those sockets. + // Do this in a separate loop to try to induce a time window to hit races. + for (size_t i = 0; i < sockets.size(); i++) { + gpr_log(GPR_DEBUG, "non-blocking accept then close on %d", sockets[i]); + ASSERT_TRUE(accept(sockets[i], nullptr, nullptr) == INVALID_SOCKET) + << "Accept on dummy socket unexpectedly accepted actual connection."; + ASSERT_TRUE(WSAGetLastError() == WSAEWOULDBLOCK) + << "OpenAndCloseSocketsStressLoop accept on socket " + + std::to_string(sockets[i]) + + " failed in " + "an unexpected way. " + "WSA error: " + + std::to_string(WSAGetLastError()) + + ". Socket use-after-close bugs are likely."; + ASSERT_TRUE(closesocket(sockets[i]) != SOCKET_ERROR) + << "Failed to close socket: " + std::to_string(sockets[i]) + + ". WSA error: " + std::to_string(WSAGetLastError()); + } + } + return; +} +#else void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { // The goal of this loop is to catch socket // "use after close" bugs within the c-ares resolver by acting @@ -311,6 +376,7 @@ void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { } } } +#endif void CheckResolverResultLocked(void* argsp, grpc_error* err) { EXPECT_EQ(err, GRPC_ERROR_NONE); @@ -372,9 +438,9 @@ void RunResolvesRelevantRecordsTest(void (*OnDoneLocked)(void* arg, args.expected_lb_policy = FLAGS_expected_lb_policy; // maybe build the address with an authority char* whole_uri = nullptr; - GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", - FLAGS_local_dns_server_address.c_str(), - FLAGS_target_name.c_str())); + GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); // create resolver and resolve grpc_core::OrphanablePtr resolver = grpc_core::ResolverRegistry::CreateResolver(whole_uri, nullptr, diff --git a/test/cpp/naming/resolver_component_tests_runner.py b/test/cpp/naming/resolver_component_tests_runner.py index 69386ebeb03..1873eec35bd 100755 --- a/test/cpp/naming/resolver_component_tests_runner.py +++ b/test/cpp/naming/resolver_component_tests_runner.py @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) @@ -112,6 +119,18 @@ wait_until_dns_server_is_up(args, dns_server_subprocess_output) num_test_failures = 0 +test_runner_log('Run test with target: %s' % 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') +current_test_subprocess = subprocess.Popen([ + args.test_bin_path, + '--target_name', 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.', + '--expected_addrs', '5.5.5.5:443,False', + '--expected_chosen_service_config', '', + '--expected_lb_policy', '', + '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port]) +current_test_subprocess.communicate() +if current_test_subprocess.returncode != 0: + num_test_failures += 1 + test_runner_log('Run test with target: %s' % 'srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') current_test_subprocess = subprocess.Popen([ args.test_bin_path, diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 6c4f89d09b8..3c51a00c7b1 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -1,5 +1,13 @@ resolver_tests_common_zone_name: resolver-tests-version-4.grpctestingexp. resolver_component_tests: +- expected_addrs: + - {address: '5.5.5.5:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: no-srv-ipv4-single-target + records: + no-srv-ipv4-single-target: + - {TTL: '2100', data: 5.5.5.5, type: A} - expected_addrs: - {address: '1.2.3.4:1234', is_balancer: true} expected_chosen_service_config: null diff --git a/test/cpp/qps/benchmark_config.cc b/test/cpp/qps/benchmark_config.cc index a4fd9de8206..5fd0f000389 100644 --- a/test/cpp/qps/benchmark_config.cc +++ b/test/cpp/qps/benchmark_config.cc @@ -22,6 +22,8 @@ #include #include +#include "test/cpp/util/test_credentials_provider.h" + DEFINE_bool(enable_log_reporter, true, "Enable reporting of benchmark results through GprLog"); @@ -44,6 +46,10 @@ DEFINE_string(rpc_reporter_server_address, "", DEFINE_bool(enable_rpc_reporter, false, "Enable use of RPC reporter"); +DEFINE_string( + rpc_reporter_credential_type, grpc::testing::kInsecureCredentialsType, + "Credential type for communication to the QPS benchmark report server"); + // In some distros, gflags is in the namespace google, and in some others, // in gflags. This hack is enabling us to find both. namespace google {} @@ -65,11 +71,14 @@ static std::shared_ptr InitBenchmarkReporters() { new JsonReporter("JsonReporter", FLAGS_scenario_result_file))); } if (FLAGS_enable_rpc_reporter) { + ChannelArguments channel_args; + std::shared_ptr channel_creds = + testing::GetCredentialsProvider()->GetChannelCredentials( + FLAGS_rpc_reporter_credential_type, &channel_args); GPR_ASSERT(!FLAGS_rpc_reporter_server_address.empty()); composite_reporter->add(std::unique_ptr(new RpcReporter( - "RpcReporter", - grpc::CreateChannel(FLAGS_rpc_reporter_server_address, - grpc::InsecureChannelCredentials())))); + "RpcReporter", grpc::CreateChannel(FLAGS_rpc_reporter_server_address, + channel_creds)))); } return std::shared_ptr(composite_reporter); diff --git a/test/cpp/qps/client.h b/test/cpp/qps/client.h index 9d58ea8882a..9d7469c9b57 100644 --- a/test/cpp/qps/client.h +++ b/test/cpp/qps/client.h @@ -19,6 +19,8 @@ #ifndef TEST_QPS_CLIENT_H #define TEST_QPS_CLIENT_H +#include + #include #include #include @@ -34,6 +36,7 @@ #include "src/proto/grpc/testing/benchmark_service.grpc.pb.h" #include "src/proto/grpc/testing/payloads.pb.h" +#include "src/core/lib/gpr/env.h" #include "src/cpp/util/core_stats.h" #include "test/cpp/qps/histogram.h" #include "test/cpp/qps/interarrival.h" @@ -441,9 +444,24 @@ class ClientImpl : public Client { std::unique_ptr WaitForReady() { return std::unique_ptr(new std::thread([this]() { if (!is_inproc_) { - GPR_ASSERT(channel_->WaitForConnected( - gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), - gpr_time_from_seconds(10, GPR_TIMESPAN)))); + int connect_deadline = 10; + /* Allow optionally overriding connect_deadline in order + * to deal with benchmark environments in which the server + * can take a long time to become ready. */ + char* channel_connect_timeout_str = + gpr_getenv("QPS_WORKER_CHANNEL_CONNECT_TIMEOUT"); + if (channel_connect_timeout_str != nullptr && + strcmp(channel_connect_timeout_str, "") != 0) { + connect_deadline = atoi(channel_connect_timeout_str); + } + gpr_log(GPR_INFO, + "Waiting for up to %d seconds for the channel %p to connect", + connect_deadline, channel_.get()); + gpr_free(channel_connect_timeout_str); + GPR_ASSERT(channel_->WaitForConnected(gpr_time_add( + gpr_now(GPR_CLOCK_REALTIME), + gpr_time_from_seconds(connect_deadline, GPR_TIMESPAN)))); + gpr_log(GPR_INFO, "Channel %p connected!", channel_.get()); } })); } diff --git a/test/cpp/thread_manager/thread_manager_test.cc b/test/cpp/thread_manager/thread_manager_test.cc index 7a95a9f17da..99de5a3e010 100644 --- a/test/cpp/thread_manager/thread_manager_test.cc +++ b/test/cpp/thread_manager/thread_manager_test.cc @@ -30,30 +30,44 @@ #include "test/cpp/util/test_config.h" namespace grpc { + +struct ThreadManagerTestSettings { + // The min number of pollers that SHOULD be active in ThreadManager + int min_pollers; + // The max number of pollers that could be active in ThreadManager + int max_pollers; + // The sleep duration in PollForWork() function to simulate "polling" + int poll_duration_ms; + // The sleep duration in DoWork() function to simulate "work" + int work_duration_ms; + // Max number of times PollForWork() is called before shutting down + int max_poll_calls; +}; + class ThreadManagerTest final : public grpc::ThreadManager { public: - ThreadManagerTest() - : ThreadManager(kMinPollers, kMaxPollers), + ThreadManagerTest(const char* name, grpc_resource_quota* rq, + const ThreadManagerTestSettings& settings) + : ThreadManager(name, rq, settings.min_pollers, settings.max_pollers), + settings_(settings), num_do_work_(0), num_poll_for_work_(0), num_work_found_(0) {} grpc::ThreadManager::WorkStatus PollForWork(void** tag, bool* ok) override; - void DoWork(void* tag, bool ok) override; - void PerformTest(); + void DoWork(void* tag, bool ok, bool resources) override; + + // Get number of times PollForWork() returned WORK_FOUND + int GetNumWorkFound(); + // Get number of times DoWork() was called + int GetNumDoWork(); private: void SleepForMs(int sleep_time_ms); - static const int kMinPollers = 2; - static const int kMaxPollers = 10; - - static const int kPollingTimeoutMsec = 10; - static const int kDoWorkDurationMsec = 1; - - // PollForWork will return SHUTDOWN after these many number of invocations - static const int kMaxNumPollForWork = 50; + ThreadManagerTestSettings settings_; + // Counters gpr_atm num_do_work_; // Number of calls to DoWork gpr_atm num_poll_for_work_; // Number of calls to PollForWork gpr_atm num_work_found_; // Number of times WORK_FOUND was returned @@ -69,54 +83,117 @@ void ThreadManagerTest::SleepForMs(int duration_ms) { grpc::ThreadManager::WorkStatus ThreadManagerTest::PollForWork(void** tag, bool* ok) { int call_num = gpr_atm_no_barrier_fetch_add(&num_poll_for_work_, 1); - - if (call_num >= kMaxNumPollForWork) { + if (call_num >= settings_.max_poll_calls) { Shutdown(); return SHUTDOWN; } - // Simulate "polling for work" by sleeping for sometime - SleepForMs(kPollingTimeoutMsec); - + SleepForMs(settings_.poll_duration_ms); // Simulate "polling" duration *tag = nullptr; *ok = true; - // Return timeout roughly 1 out of every 3 calls + // Return timeout roughly 1 out of every 3 calls just to make the test a bit + // more interesting if (call_num % 3 == 0) { return TIMEOUT; - } else { - gpr_atm_no_barrier_fetch_add(&num_work_found_, 1); - return WORK_FOUND; } + + gpr_atm_no_barrier_fetch_add(&num_work_found_, 1); + return WORK_FOUND; } -void ThreadManagerTest::DoWork(void* tag, bool ok) { +void ThreadManagerTest::DoWork(void* tag, bool ok, bool resources) { gpr_atm_no_barrier_fetch_add(&num_do_work_, 1); - SleepForMs(kDoWorkDurationMsec); // Simulate doing work by sleeping + SleepForMs(settings_.work_duration_ms); // Simulate work by sleeping } -void ThreadManagerTest::PerformTest() { - // Initialize() starts the ThreadManager - Initialize(); - - // Wait for all the threads to gracefully terminate - Wait(); +int ThreadManagerTest::GetNumWorkFound() { + return static_cast(gpr_atm_no_barrier_load(&num_work_found_)); +} - // The number of times DoWork() was called is equal to the number of times - // WORK_FOUND was returned - gpr_log(GPR_DEBUG, "DoWork() called %" PRIdPTR " times", - gpr_atm_no_barrier_load(&num_do_work_)); - GPR_ASSERT(gpr_atm_no_barrier_load(&num_do_work_) == - gpr_atm_no_barrier_load(&num_work_found_)); +int ThreadManagerTest::GetNumDoWork() { + return static_cast(gpr_atm_no_barrier_load(&num_do_work_)); } } // namespace grpc +// Test that the number of times DoWork() is called is equal to the number of +// times PollForWork() returned WORK_FOUND +static void TestPollAndWork() { + grpc_resource_quota* rq = grpc_resource_quota_create("Test-poll-and-work"); + grpc::ThreadManagerTestSettings settings = { + 2 /* min_pollers */, 10 /* max_pollers */, 10 /* poll_duration_ms */, + 1 /* work_duration_ms */, 50 /* max_poll_calls */}; + + grpc::ThreadManagerTest test_thread_mgr("TestThreadManager", rq, settings); + grpc_resource_quota_unref(rq); + + test_thread_mgr.Initialize(); // Start the thread manager + test_thread_mgr.Wait(); // Wait for all threads to finish + + // Verify that The number of times DoWork() was called is equal to the number + // of times WORK_FOUND was returned + gpr_log(GPR_DEBUG, "DoWork() called %d times", + test_thread_mgr.GetNumDoWork()); + GPR_ASSERT(test_thread_mgr.GetNumDoWork() == + test_thread_mgr.GetNumWorkFound()); +} + +static void TestThreadQuota() { + const int kMaxNumThreads = 3; + grpc_resource_quota* rq = grpc_resource_quota_create("Test-thread-quota"); + grpc_resource_quota_set_max_threads(rq, kMaxNumThreads); + + // Set work_duration_ms to be much greater than poll_duration_ms. This way, + // the thread manager will be forced to create more 'polling' threads to + // honor the min_pollers guarantee + grpc::ThreadManagerTestSettings settings = { + 1 /* min_pollers */, 1 /* max_pollers */, 1 /* poll_duration_ms */, + 10 /* work_duration_ms */, 50 /* max_poll_calls */}; + + // Create two thread managers (but with same resource quota). This means + // that the max number of active threads across BOTH the thread managers + // cannot be greater than kMaxNumthreads + grpc::ThreadManagerTest test_thread_mgr_1("TestThreadManager-1", rq, + settings); + grpc::ThreadManagerTest test_thread_mgr_2("TestThreadManager-2", rq, + settings); + // It is ok to unref resource quota before starting thread managers. + grpc_resource_quota_unref(rq); + + // Start both thread managers + test_thread_mgr_1.Initialize(); + test_thread_mgr_2.Initialize(); + + // Wait for both to finish + test_thread_mgr_1.Wait(); + test_thread_mgr_2.Wait(); + + // Now verify that the total number of active threads in either thread manager + // never exceeds kMaxNumThreads + // + // NOTE: Actually the total active threads across *both* thread managers at + // any point of time never exceeds kMaxNumThreads but unfortunately there is + // no easy way to verify it (i.e we can't just do (max1 + max2 <= k)) + // Its okay to not test this case here. The resource quota c-core tests + // provide enough coverage to resource quota object with multiple resource + // users + int max1 = test_thread_mgr_1.GetMaxActiveThreadsSoFar(); + int max2 = test_thread_mgr_2.GetMaxActiveThreadsSoFar(); + gpr_log( + GPR_DEBUG, + "MaxActiveThreads in TestThreadManager_1: %d, TestThreadManager_2: %d", + max1, max2); + GPR_ASSERT(max1 <= kMaxNumThreads && max2 <= kMaxNumThreads); +} + int main(int argc, char** argv) { std::srand(std::time(nullptr)); - grpc::testing::InitTest(&argc, &argv, true); - grpc::ThreadManagerTest test_rpc_manager; - test_rpc_manager.PerformTest(); + grpc_init(); + + TestPollAndWork(); + TestThreadQuota(); + grpc_shutdown(); return 0; } diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index 9b42bb28b1d..477862a0ee1 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -177,6 +177,7 @@ grpc_cc_test( "//:grpc++_reflection", "//src/proto/grpc/testing:echo_messages_proto", "//src/proto/grpc/testing:echo_proto", + "//test/core/end2end:ssl_test_data", "//test/core/util:grpc_test_util", ], ) @@ -268,27 +269,15 @@ grpc_cc_test( grpc_cc_binary( name = "grpc_cli", srcs = [ - "cli_call.cc", - "cli_call.h", - "cli_credentials.cc", - "cli_credentials.h", - "config_grpc_cli.h", "grpc_cli.cc", - "grpc_tool.cc", - "grpc_tool.h", - "proto_file_parser.cc", - "proto_file_parser.h", - "proto_reflection_descriptor_database.cc", - "proto_reflection_descriptor_database.h", - "service_describer.cc", - "service_describer.h", - "test_config.h", - "test_config_cc.cc", ], external_deps = [ "gflags", ], deps = [ + ":grpc_cli_libs", + ":grpc++_proto_reflection_desc_db", + ":test_config", "//:grpc++", "//src/proto/grpc/reflection/v1alpha:reflection_proto", ], diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index aa4eafb7569..acf4ef8ef10 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -20,45 +20,123 @@ #include -DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); +DEFINE_bool( + enable_ssl, false, + "Whether to use ssl/tls. Deprecated. Use --channel_creds_type=ssl."); +DEFINE_bool(use_auth, false, + "Whether to create default google credentials. Deprecated. Use " + "--channel_creds_type=gdc."); DEFINE_string( access_token, "", "The access token that will be sent to the server to authenticate RPCs."); +DEFINE_string( + ssl_target, "", + "If not empty, treat the server host name as this for ssl/tls certificate " + "validation."); +DEFINE_string( + channel_creds_type, "", + "The channel creds type: insecure, ssl, gdc (Google Default Credentials) " + "or alts."); namespace grpc { namespace testing { -std::shared_ptr CliCredentials::GetCredentials() +grpc::string CliCredentials::GetDefaultChannelCredsType() const { + // Compatibility logic for --enable_ssl. + if (FLAGS_enable_ssl) { + fprintf(stderr, + "warning: --enable_ssl is deprecated. Use " + "--channel_creds_type=ssl.\n"); + return "ssl"; + } + // Compatibility logic for --use_auth. + if (FLAGS_access_token.empty() && FLAGS_use_auth) { + fprintf(stderr, + "warning: --use_auth is deprecated. Use " + "--channel_creds_type=gdc.\n"); + return "gdc"; + } + return "insecure"; +} + +std::shared_ptr +CliCredentials::GetChannelCredentials() const { + if (FLAGS_channel_creds_type.compare("insecure") == 0) { + return grpc::InsecureChannelCredentials(); + } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { + return grpc::SslCredentials(grpc::SslCredentialsOptions()); + } else if (FLAGS_channel_creds_type.compare("gdc") == 0) { + return grpc::GoogleDefaultCredentials(); + } else if (FLAGS_channel_creds_type.compare("alts") == 0) { + return grpc::experimental::AltsCredentials( + grpc::experimental::AltsCredentialsOptions()); + } + fprintf(stderr, + "--channel_creds_type=%s invalid; must be insecure, ssl, gdc or " + "alts.\n", + FLAGS_channel_creds_type.c_str()); + return std::shared_ptr(); +} + +std::shared_ptr CliCredentials::GetCallCredentials() const { if (!FLAGS_access_token.empty()) { if (FLAGS_use_auth) { fprintf(stderr, "warning: use_auth is ignored when access_token is provided."); } - - return grpc::CompositeChannelCredentials( - grpc::SslCredentials(grpc::SslCredentialsOptions()), - grpc::AccessTokenCredentials(FLAGS_access_token)); + return grpc::AccessTokenCredentials(FLAGS_access_token); } + return std::shared_ptr(); +} - if (FLAGS_use_auth) { - return grpc::GoogleDefaultCredentials(); +std::shared_ptr CliCredentials::GetCredentials() + const { + if (FLAGS_channel_creds_type.empty()) { + FLAGS_channel_creds_type = GetDefaultChannelCredsType(); + } else if (FLAGS_enable_ssl && FLAGS_channel_creds_type.compare("ssl") != 0) { + fprintf(stderr, + "warning: ignoring --enable_ssl because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); + } else if (FLAGS_use_auth && FLAGS_channel_creds_type.compare("gdc") != 0) { + fprintf(stderr, + "warning: ignoring --use_auth because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); } - - if (FLAGS_enable_ssl) { - return grpc::SslCredentials(grpc::SslCredentialsOptions()); + // Legacy transport upgrade logic for insecure requests. + if (!FLAGS_access_token.empty() && + FLAGS_channel_creds_type.compare("insecure") == 0) { + fprintf(stderr, + "warning: --channel_creds_type=insecure upgraded to ssl because " + "an access token was provided.\n"); + FLAGS_channel_creds_type = "ssl"; } - - return grpc::InsecureChannelCredentials(); + std::shared_ptr channel_creds = + GetChannelCredentials(); + // Composite any call-type credentials on top of the base channel. + std::shared_ptr call_creds = GetCallCredentials(); + return (channel_creds == nullptr || call_creds == nullptr) + ? channel_creds + : grpc::CompositeChannelCredentials(channel_creds, call_creds); } const grpc::string CliCredentials::GetCredentialUsage() const { - return " --enable_ssl ; Set whether to use tls\n" + return " --enable_ssl ; Set whether to use ssl (deprecated)\n" " --use_auth ; Set whether to create default google" " credentials\n" " --access_token ; Set the access token in metadata," - " overrides --use_auth\n"; + " overrides --use_auth\n" + " --ssl_target ; Set server host for ssl validation\n" + " --channel_creds_type ; Set to insecure, ssl, gdc, or alts\n"; } + +const grpc::string CliCredentials::GetSslTargetNameOverride() const { + bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || + FLAGS_channel_creds_type.compare("gdc") == 0; + return use_ssl ? FLAGS_ssl_target : ""; +} + } // namespace testing } // namespace grpc diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index b1358e77d8b..4636d3ca149 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -28,8 +28,22 @@ namespace testing { class CliCredentials { public: virtual ~CliCredentials() {} - virtual std::shared_ptr GetCredentials() const; + std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; + virtual const grpc::string GetSslTargetNameOverride() const; + + protected: + // Returns the appropriate channel_creds_type value for the set of legacy + // flag arguments. + virtual grpc::string GetDefaultChannelCredsType() const; + // Returns the base transport channel credentials. Child classes can override + // to support additional channel_creds_types unknown to this base class. + virtual std::shared_ptr GetChannelCredentials() + const; + // Returns call credentials to composite onto the base transport channel + // credentials. Child classes can override to support additional + // authentication flags unknown to this base class. + virtual std::shared_ptr GetCallCredentials() const; }; } // namespace testing diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 840ca07d2bf..ccc60cca27a 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -206,6 +206,15 @@ void ReadResponse(CliCall* call, const grpc::string& method_name, } } +std::shared_ptr CreateCliChannel( + const grpc::string& server_address, const CliCredentials& cred) { + grpc::ChannelArguments args; + if (!cred.GetSslTargetNameOverride().empty()) { + args.SetSslTargetNameOverride(cred.GetSslTargetNameOverride()); + } + return grpc::CreateCustomChannel(server_address, cred.GetCredentials(), args); +} + struct Command { const char* command; std::function channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); grpc::ProtoReflectionDescriptorDatabase desc_db(channel); grpc::protobuf::DescriptorPool desc_pool(&desc_db); @@ -422,7 +431,7 @@ bool GrpcTool::PrintType(int argc, const char** argv, grpc::string server_address(argv[0]); std::shared_ptr channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); grpc::ProtoReflectionDescriptorDatabase desc_db(channel); grpc::protobuf::DescriptorPool desc_pool(&desc_db); @@ -469,7 +478,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, bool print_mode = false; std::shared_ptr channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); if (!FLAGS_binary_input || !FLAGS_binary_output) { parser.reset( @@ -820,7 +829,7 @@ bool GrpcTool::ParseMessage(int argc, const char** argv, if (!FLAGS_binary_input || !FLAGS_binary_output) { std::shared_ptr channel = - grpc::CreateChannel(server_address, cred.GetCredentials()); + CreateCliChannel(server_address, cred); parser.reset( new grpc::testing::ProtoFileParser(FLAGS_remotedb ? channel : nullptr, FLAGS_proto_path, FLAGS_protofiles)); diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 6574d1bb441..3aae090e818 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -35,6 +35,7 @@ #include "src/core/lib/gpr/env.h" #include "src/proto/grpc/testing/echo.grpc.pb.h" #include "src/proto/grpc/testing/echo.pb.h" +#include "test/core/end2end/data/ssl_test_data.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" #include "test/cpp/util/cli_credentials.h" @@ -80,6 +81,9 @@ using grpc::testing::EchoResponse; " peer: \"peer\"\n" \ "}\n\n" +DECLARE_string(channel_creds_type); +DECLARE_string(ssl_target); + namespace grpc { namespace testing { @@ -97,10 +101,19 @@ const int kServerDefaultResponseStreamsToSend = 3; class TestCliCredentials final : public grpc::testing::CliCredentials { public: - std::shared_ptr GetCredentials() const override { - return InsecureChannelCredentials(); + TestCliCredentials(bool secure = false) : secure_(secure) {} + std::shared_ptr GetChannelCredentials() + const override { + if (!secure_) { + return InsecureChannelCredentials(); + } + SslCredentialsOptions ssl_opts = {test_root_cert, "", ""}; + return SslCredentials(grpc::SslCredentialsOptions(ssl_opts)); } const grpc::string GetCredentialUsage() const override { return ""; } + + private: + const bool secure_; }; bool PrintStream(std::stringstream* ss, const grpc::string& output) { @@ -206,13 +219,24 @@ class GrpcToolTest : public ::testing::Test { // SetUpServer cannot be used with EXPECT_EXIT. grpc_pick_unused_port_or_die() // uses atexit() to free chosen ports, and it will spawn a new thread in // resolve_address_posix.c:192 at exit time. - const grpc::string SetUpServer() { + const grpc::string SetUpServer(bool secure = false) { std::ostringstream server_address; int port = grpc_pick_unused_port_or_die(); server_address << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address.str(), InsecureServerCredentials()); + std::shared_ptr creds; + if (secure) { + SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key, + test_server1_cert}; + SslServerCredentialsOptions ssl_opts; + ssl_opts.pem_root_certs = ""; + ssl_opts.pem_key_cert_pairs.push_back(pkcp); + creds = SslServerCredentials(ssl_opts); + } else { + creds = InsecureServerCredentials(); + } + builder.AddListeningPort(server_address.str(), creds); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); return server_address.str(); @@ -743,6 +767,29 @@ TEST_F(GrpcToolTest, CallCommandWithBadMetadata) { gpr_free(test_srcdir); } +TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { + const grpc::string server_address = SetUpServer(true); + + // Test input "grpc_cli ls localhost: --channel_creds_type=ssl + // --ssl_target=z.test.google.fr" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli", "ls", server_address.c_str()}; + FLAGS_l = false; + FLAGS_channel_creds_type = "ssl"; + FLAGS_ssl_target = "z.test.google.fr"; + EXPECT_TRUE( + 0 == GrpcToolMainLib( + ArraySize(argv), argv, TestCliCredentials(true), + std::bind(PrintStream, &output_stream, std::placeholders::_1))); + EXPECT_TRUE(0 == strcmp(output_stream.str().c_str(), + "grpc.testing.EchoTestService\n" + "grpc.reflection.v1alpha.ServerReflection\n")); + + FLAGS_channel_creds_type = ""; + FLAGS_ssl_target = ""; + ShutdownServer(); +} + } // namespace testing } // namespace grpc diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat index 8eb3b201b13..7658156be1d 100644 --- a/test/distrib/cpp/run_distrib_test_cmake.bat +++ b/test/distrib/cpp/run_distrib_test_cmake.bat @@ -39,7 +39,7 @@ cd ../../.. cd third_party/protobuf/cmake mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. +cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DZLIB_ROOT=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. cmake --build . --config Release --target install || goto :error cd ../../../.. diff --git a/test/distrib/cpp/run_distrib_test_routeguide.sh b/test/distrib/cpp/run_distrib_test_routeguide.sh index dc69ab83773..a81692d920c 100755 --- a/test/distrib/cpp/run_distrib_test_routeguide.sh +++ b/test/distrib/cpp/run_distrib_test_routeguide.sh @@ -19,7 +19,7 @@ set -ex cd "$(dirname "$0")/../../.." cd third_party/protobuf && ./autogen.sh && \ -./configure && make -j4 && make check && make install && ldconfig +./configure && make -j4 && make install && ldconfig cd ../.. && make -j4 && make install diff --git a/third_party/BUILD b/third_party/BUILD index f06c5e9c67d..5ec919dc48d 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -8,4 +8,5 @@ exports_files([ "incremental.BUILD", "zope_interface.BUILD", "constantly.BUILD", + "cython.BUILD", ]) diff --git a/third_party/address_sorting/address_sorting.c b/third_party/address_sorting/address_sorting.c index e4f3b537992..9aee0a5419f 100644 --- a/third_party/address_sorting/address_sorting.c +++ b/third_party/address_sorting/address_sorting.c @@ -55,12 +55,17 @@ static const int kIPv6AddrScopeGlobal = 3; static address_sorting_source_addr_factory* g_current_source_addr_factory = NULL; -static int address_sorting_get_source_addr(const address_sorting_address* dest, - address_sorting_address* source) { +static bool address_sorting_get_source_addr(const address_sorting_address* dest, + address_sorting_address* source) { return g_current_source_addr_factory->vtable->get_source_addr( g_current_source_addr_factory, dest, source); } +bool address_sorting_get_source_addr_for_testing( + const address_sorting_address* dest, address_sorting_address* source) { + return address_sorting_get_source_addr(dest, source); +} + static int ipv6_prefix_match_length(const struct sockaddr_in6* sa, const struct sockaddr_in6* sb) { unsigned char* a = (unsigned char*)&sa->sin6_addr; diff --git a/third_party/address_sorting/address_sorting_windows.c b/third_party/address_sorting/address_sorting_windows.c index b2f5708649e..662a88248e0 100644 --- a/third_party/address_sorting/address_sorting_windows.c +++ b/third_party/address_sorting/address_sorting_windows.c @@ -42,14 +42,54 @@ #if defined(ADDRESS_SORTING_WINDOWS) +#include +#include +#include #include +#include +#include -/* TODO : Add address sorting functionality to work on windows. */ +static bool windows_source_addr_factory_get_source_addr( + address_sorting_source_addr_factory* factory, + const address_sorting_address* dest_addr, + address_sorting_address* source_addr) { + bool source_addr_exists = false; + SOCKET s = socket(((struct sockaddr_in6*)dest_addr)->sin6_family, SOCK_DGRAM, + IPPROTO_UDP); + if (s != INVALID_SOCKET) { + if (connect(s, (struct sockaddr*)dest_addr, (int)dest_addr->len) == 0) { + address_sorting_address found_source_addr; + memset(&found_source_addr, 0, sizeof(found_source_addr)); + found_source_addr.len = sizeof(found_source_addr.addr); + if (getsockname(s, (struct sockaddr*)&found_source_addr.addr, + (socklen_t*)&found_source_addr.len) == 0) { + source_addr_exists = true; + *source_addr = found_source_addr; + } + } + closesocket(s); + } + return source_addr_exists; +} + +static void windows_source_addr_factory_destroy( + address_sorting_source_addr_factory* self) { + free(self); +} + +static const address_sorting_source_addr_factory_vtable + windows_source_addr_factory_vtable = { + windows_source_addr_factory_get_source_addr, + windows_source_addr_factory_destroy, +}; address_sorting_source_addr_factory* address_sorting_create_source_addr_factory_for_current_platform() { - abort(); - return NULL; + address_sorting_source_addr_factory* factory = + malloc(sizeof(address_sorting_source_addr_factory)); + memset(factory, 0, sizeof(address_sorting_source_addr_factory)); + factory->vtable = &windows_source_addr_factory_vtable; + return factory; } #endif // defined(ADDRESS_SORTING_WINDOWS) diff --git a/third_party/address_sorting/include/address_sorting/address_sorting.h b/third_party/address_sorting/include/address_sorting/address_sorting.h index f11cd424b53..c58fafe3f70 100644 --- a/third_party/address_sorting/include/address_sorting/address_sorting.h +++ b/third_party/address_sorting/include/address_sorting/address_sorting.h @@ -103,6 +103,9 @@ address_sorting_family address_sorting_abstract_get_family( void address_sorting_override_source_addr_factory_for_testing( address_sorting_source_addr_factory* factory); +bool address_sorting_get_source_addr_for_testing( + const address_sorting_address* dest, address_sorting_address* source); + #ifdef __cplusplus } #endif diff --git a/third_party/cython.BUILD b/third_party/cython.BUILD new file mode 100644 index 00000000000..ce9283e2a41 --- /dev/null +++ b/third_party/cython.BUILD @@ -0,0 +1,29 @@ +# Adapted with modifications from tensorflow/third_party/cython.BUILD + +py_library( + name="cython_lib", + srcs=glob( + ["Cython/**/*.py"], + exclude=[ + "**/Tests/*.py", + ], + ) + ["cython.py"], + data=glob([ + "Cython/**/*.pyx", + "Cython/Utility/*.*", + "Cython/Includes/**/*.pxd", + ]), + srcs_version="PY2AND3", + visibility=["//visibility:public"], +) + +# May not be named "cython", since that conflicts with Cython/ on OSX +py_binary( + name="cython_binary", + srcs=["cython.py"], + main="cython.py", + srcs_version="PY2AND3", + visibility=["//visibility:public"], + deps=["cython_lib"], +) + diff --git a/third_party/protobuf b/third_party/protobuf index b5fbb742af1..48cb18e5c41 160000 --- a/third_party/protobuf +++ b/third_party/protobuf @@ -1 +1 @@ -Subproject commit b5fbb742af122b565925987e65c08957739976a7 +Subproject commit 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 diff --git a/third_party/py/BUILD b/third_party/py/BUILD new file mode 100644 index 00000000000..e69de29bb2d diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl new file mode 100644 index 00000000000..2283c573bc3 --- /dev/null +++ b/third_party/py/BUILD.tpl @@ -0,0 +1,36 @@ +# Adapted with modifications from tensorflow/third_party/py/ + +package(default_visibility=["//visibility:public"]) + +# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib +# See https://docs.python.org/3/extending/windows.html +cc_import( + name="python_lib", + interface_library=select({ + ":windows": ":python_import_lib", + # A placeholder for Unix platforms which makes --no_build happy. + "//conditions:default": "not-existing.lib", + }), + system_provided=1, +) + +cc_library( + name="python_headers", + hdrs=[":python_include"], + deps=select({ + ":windows": [":python_lib"], + "//conditions:default": [], + }), + includes=["python_include"], +) + +config_setting( + name="windows", + values={"cpu": "x64_windows"}, + visibility=["//visibility:public"], +) + +%{PYTHON_INCLUDE_GENRULE} +%{PYTHON_IMPORT_LIB_GENRULE} + + diff --git a/third_party/py/python_configure.bzl b/third_party/py/python_configure.bzl new file mode 100644 index 00000000000..2ba1e07049c --- /dev/null +++ b/third_party/py/python_configure.bzl @@ -0,0 +1,305 @@ +# Adapted with modifications from tensorflow/third_party/py/ +"""Repository rule for Python autoconfiguration. + +`python_configure` depends on the following environment variables: + + * `PYTHON_BIN_PATH`: location of python binary. + * `PYTHON_LIB_PATH`: Location of python libraries. +""" + +_BAZEL_SH = "BAZEL_SH" +_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" +_PYTHON_LIB_PATH = "PYTHON_LIB_PATH" +_PYTHON_CONFIG_REPO = "PYTHON_CONFIG_REPO" + + +def _tpl(repository_ctx, tpl, substitutions={}, out=None): + if not out: + out = tpl + repository_ctx.template(out, Label("//third_party/py:%s.tpl" % tpl), + substitutions) + + +def _fail(msg): + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) + + +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + return os_name.find("windows") != -1 + + +def _execute(repository_ctx, + cmdline, + error_msg=None, + error_details=None, + empty_stdout_fine=False): + """Executes an arbitrary shell command. + + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise + it's an error + Return: + the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + _fail("\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), error_details if error_details else "" + ])) + else: + return result + + +def _read_dir(repository_ctx, src_dir): + """Returns a string with all files in a directory. + + Finds all files inside a directory, traversing subfolders and following + symlinks. The returned string contains the full path of all files + separated by line breaks. + """ + if _is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine=True) + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + return find_result.stdout.replace("\\", "/") + else: + find_result = _execute( + repository_ctx, ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine=True) + return find_result.stdout + + +def _genrule(src_dir, genrule_name, command, outs): + """Returns a string with a genrule. + + Genrule executes the given command and produces the given outputs. + """ + return ('genrule(\n' + ' name = "' + genrule_name + '",\n' + + ' outs = [\n' + outs + '\n ],\n' + ' cmd = """\n' + + command + '\n """,\n' + ')\n') + + +def _normalize_path(path): + """Returns a path with '/' and remove the trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + + +def _symlink_genrule_for_dir(repository_ctx, + src_dir, + dest_dir, + genrule_name, + src_files=[], + dest_files=[]): + """Returns a genrule to symlink(or copy if on Windows) a set of files. + + If src_dir is passed, files will be read from the given directory; otherwise + we assume files are in src_files and dest_files + """ + if src_dir != None: + src_dir = _normalize_path(src_dir) + dest_dir = _normalize_path(dest_dir) + files = '\n'.join( + sorted(_read_dir(repository_ctx, src_dir).splitlines())) + # Create a list with the src_dir stripped to use for outputs. + dest_files = files.replace(src_dir, '').splitlines() + src_files = files.splitlines() + command = [] + outs = [] + for i in range(len(dest_files)): + if dest_files[i] != "": + # If we have only one file to link we do not want to use the dest_dir, as + # $(@D) will include the full path to the file. + dest = '$(@D)/' + dest_dir + dest_files[i] if len( + dest_files) != 1 else '$(@D)/' + dest_files[i] + # On Windows, symlink is not supported, so we just copy all the files. + cmd = 'cp -f' if _is_windows(repository_ctx) else 'ln -s' + command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) + outs.append(' "' + dest_dir + dest_files[i] + '",') + return _genrule(src_dir, genrule_name, " && ".join(command), + "\n".join(outs)) + + +def _get_python_bin(repository_ctx): + """Gets the python bin path.""" + python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) + if python_bin != None: + return python_bin + python_bin_path = repository_ctx.which("python") + if python_bin_path != None: + return str(python_bin_path) + _fail("Cannot find python in PATH, please make sure " + + "python is installed and add its directory in PATH, or --define " + + "%s='/something/else'.\nPATH=%s" % + (_PYTHON_BIN_PATH, repository_ctx.os.environ.get("PATH", ""))) + + +def _get_bash_bin(repository_ctx): + """Gets the bash bin path.""" + bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) + if bash_bin != None: + return bash_bin + else: + bash_bin_path = repository_ctx.which("bash") + if bash_bin_path != None: + return str(bash_bin_path) + else: + _fail( + "Cannot find bash in PATH, please make sure " + + "bash is installed and add its directory in PATH, or --define " + + "%s='/path/to/bash'.\nPATH=%s" % + (_BAZEL_SH, repository_ctx.os.environ.get("PATH", ""))) + + +def _get_python_lib(repository_ctx, python_bin): + """Gets the python lib path.""" + python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) + if python_lib != None: + return python_lib + print_lib = ( + "<=1:\n" + + " print(paths[0])\n" + "END") + cmd = '%s - %s' % (python_bin, print_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + return result.stdout.strip('\n') + + +def _check_python_lib(repository_ctx, python_lib): + """Checks the python lib path.""" + cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("Invalid python library path: %s" % python_lib) + + +def _check_python_bin(repository_ctx, python_bin): + """Checks the python bin path.""" + cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) + result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) + if result.return_code == 1: + _fail("--define %s='%s' is not executable. Is it the python binary?" % + (_PYTHON_BIN_PATH, python_bin)) + + +def _get_python_include(repository_ctx, python_bin): + """Gets the python include path.""" + result = _execute( + repository_ctx, [ + python_bin, "-c", 'from __future__ import print_function;' + + 'from distutils import sysconfig;' + + 'print(sysconfig.get_python_inc())' + ], + error_msg="Problem getting python include path.", + error_details=( + "Is the Python binary path set up right? " + "(See ./configure or " + + _PYTHON_BIN_PATH + ".) " + "Is distutils installed?")) + return result.stdout.splitlines()[0] + + +def _get_python_import_lib_name(repository_ctx, python_bin): + """Get Python import library name (pythonXY.lib) on Windows.""" + result = _execute( + repository_ctx, [ + python_bin, "-c", + 'import sys;' + 'print("python" + str(sys.version_info[0]) + ' + + ' str(sys.version_info[1]) + ".lib")' + ], + error_msg="Problem getting python import library.", + error_details=("Is the Python binary path set up right? " + + "(See ./configure or " + _PYTHON_BIN_PATH + ".) ")) + return result.stdout.splitlines()[0] + + +def _create_local_python_repository(repository_ctx): + """Creates the repository containing files set up to build with Python.""" + python_bin = _get_python_bin(repository_ctx) + _check_python_bin(repository_ctx, python_bin) + python_lib = _get_python_lib(repository_ctx, python_bin) + _check_python_lib(repository_ctx, python_lib) + python_include = _get_python_include(repository_ctx, python_bin) + python_include_rule = _symlink_genrule_for_dir( + repository_ctx, python_include, 'python_include', 'python_include') + python_import_lib_genrule = "" + # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib + # See https://docs.python.org/3/extending/windows.html + if _is_windows(repository_ctx): + python_include = _normalize_path(python_include) + python_import_lib_name = _get_python_import_lib_name( + repository_ctx, python_bin) + python_import_lib_src = python_include.rsplit( + '/', 1)[0] + "/libs/" + python_import_lib_name + python_import_lib_genrule = _symlink_genrule_for_dir( + repository_ctx, None, '', 'python_import_lib', + [python_import_lib_src], [python_import_lib_name]) + _tpl( + repository_ctx, "BUILD", { + "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, + "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, + }) + + +def _create_remote_python_repository(repository_ctx, remote_config_repo): + """Creates pointers to a remotely configured repo set up to build with Python. + """ + _tpl(repository_ctx, "remote.BUILD", { + "%{REMOTE_PYTHON_REPO}": remote_config_repo, + }, "BUILD") + + +def _python_autoconf_impl(repository_ctx): + """Implementation of the python_autoconf repository rule.""" + if _PYTHON_CONFIG_REPO in repository_ctx.os.environ: + _create_remote_python_repository( + repository_ctx, repository_ctx.os.environ[_PYTHON_CONFIG_REPO]) + else: + _create_local_python_repository(repository_ctx) + + +python_configure = repository_rule( + implementation=_python_autoconf_impl, + environ=[ + _BAZEL_SH, + _PYTHON_BIN_PATH, + _PYTHON_LIB_PATH, + _PYTHON_CONFIG_REPO, + ], +) +"""Detects and configures the local Python. + +Add the following to your WORKSPACE FILE: + +```python +python_configure(name = "local_config_python") +``` + +Args: + name: A unique name for this workspace rule. +""" + diff --git a/third_party/py/remote.BUILD.tpl b/third_party/py/remote.BUILD.tpl new file mode 100644 index 00000000000..1bfe1f0bf65 --- /dev/null +++ b/third_party/py/remote.BUILD.tpl @@ -0,0 +1,10 @@ +# Adapted with modifications from tensorflow/third_party/py/ + +package(default_visibility=["//visibility:public"]) + +alias( + name="python_headers", + actual="%{REMOTE_PYTHON_REPO}:python_headers", +) + + diff --git a/third_party/toolchains/BUILD b/third_party/toolchains/BUILD index 29fc4fdaf3c..02cd87a7b9b 100644 --- a/third_party/toolchains/BUILD +++ b/third_party/toolchains/BUILD @@ -42,7 +42,7 @@ platform( } properties: { name: "gceMachineType" # Small machines for majority of tests. - value: "n1-standard-1" + value: "n1-highmem-2" } properties: { name: "gceMachineType_LARGE" # Large machines for a small set of resource-consuming tests such as combiner_tests under TSAN. diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index 5c781761ba1..15495917467 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -68,7 +68,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.5.2 +pip install protobuf==3.6.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null diff --git a/tools/distrib/check_copyright.py b/tools/distrib/check_copyright.py index e6388251cf6..77f602ad1fc 100755 --- a/tools/distrib/check_copyright.py +++ b/tools/distrib/check_copyright.py @@ -100,6 +100,10 @@ _EXEMPT = frozenset(( # Gradle wrappers used to build for Android 'examples/android/helloworld/gradlew.bat', 'src/android/test/interop/gradlew.bat', + + # Designer-generated source + 'examples/csharp/HelloworldXamarin/Droid/Resources/Resource.designer.cs', + 'examples/csharp/HelloworldXamarin/iOS/ViewController.designer.cs', )) RE_YEAR = r'Copyright (?P[0-9]+\-)?(?P[0-9]+) ([Tt]he )?gRPC [Aa]uthors(\.|)' diff --git a/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc b/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc deleted file mode 100644 index ba93621e4ff..00000000000 --- a/tools/distrib/python/grpcio_tools/grpc_tools/protobuf_generated_well_known_types_embed.cc +++ /dev/null @@ -1,328 +0,0 @@ -// Copyright 2017 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// HACK: Embed the generated well_known_types_js.cc to make -// grpc-tools python package compilation easy. -#include -struct FileToc well_known_types_js[] = { -{"any.js", - "// Protocol Buffers - Google's data interchange format\n" - "// Copyright 2008 Google Inc. All rights reserved.\n" - "// https://developers.google.com/protocol-buffers/\n" - "//\n" - "// Redistribution and use in source and binary forms, with or without\n" - "// modification, are permitted provided that the following conditions are\n" - "// met:\n" - "//\n" - "// * Redistributions of source code must retain the above copyright\n" - "// notice, this list of conditions and the following disclaimer.\n" - "// * Redistributions in binary form must reproduce the above\n" - "// copyright notice, this list of conditions and the following disclaimer\n" - "// in the documentation and/or other materials provided with the\n" - "// distribution.\n" - "// * Neither the name of Google Inc. nor the names of its\n" - "// contributors may be used to endorse or promote products derived from\n" - "// this software without specific prior written permission.\n" - "//\n" - "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" - "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" - "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" - "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" - "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" - "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" - "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" - "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" - "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" - "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" - "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" - "\n" - "/* This code will be inserted into generated code for\n" - " * google/protobuf/any.proto. */\n" - "\n" - "/**\n" - " * Returns the type name contained in this instance, if any.\n" - " * @return {string|undefined}\n" - " */\n" - "proto.google.protobuf.Any.prototype.getTypeName = function() {\n" - " return this.getTypeUrl().split('/').pop();\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Packs the given message instance into this Any.\n" - " * @param {!Uint8Array} serialized The serialized data to pack.\n" - " * @param {string} name The type name of this message object.\n" - " * @param {string=} opt_typeUrlPrefix the type URL prefix.\n" - " */\n" - "proto.google.protobuf.Any.prototype.pack = function(serialized, name,\n" - " opt_typeUrlPrefix) {\n" - " if (!opt_typeUrlPrefix) {\n" - " opt_typeUrlPrefix = 'type.googleapis.com/';\n" - " }\n" - "\n" - " if (opt_typeUrlPrefix.substr(-1) != '/') {\n" - " this.setTypeUrl(opt_typeUrlPrefix + '/' + name);\n" - " } else {\n" - " this.setTypeUrl(opt_typeUrlPrefix + name);\n" - " }\n" - "\n" - " this.setValue(serialized);\n" - "};\n" - "\n" - "\n" - "/**\n" - " * @template T\n" - " * Unpacks this Any into the given message object.\n" - " * @param {function(Uint8Array):T} deserialize Function that will deserialize\n" - " * the binary data properly.\n" - " * @param {string} name The expected type name of this message object.\n" - " * @return {?T} If the name matched the expected name, returns the deserialized\n" - " * object, otherwise returns undefined.\n" - " */\n" - "proto.google.protobuf.Any.prototype.unpack = function(deserialize, name) {\n" - " if (this.getTypeName() == name) {\n" - " return deserialize(this.getValue_asU8());\n" - " } else {\n" - " return null;\n" - " }\n" - "};\n" -}, -{"struct.js", - "// Protocol Buffers - Google's data interchange format\n" - "// Copyright 2008 Google Inc. All rights reserved.\n" - "// https://developers.google.com/protocol-buffers/\n" - "//\n" - "// Redistribution and use in source and binary forms, with or without\n" - "// modification, are permitted provided that the following conditions are\n" - "// met:\n" - "//\n" - "// * Redistributions of source code must retain the above copyright\n" - "// notice, this list of conditions and the following disclaimer.\n" - "// * Redistributions in binary form must reproduce the above\n" - "// copyright notice, this list of conditions and the following disclaimer\n" - "// in the documentation and/or other materials provided with the\n" - "// distribution.\n" - "// * Neither the name of Google Inc. nor the names of its\n" - "// contributors may be used to endorse or promote products derived from\n" - "// this software without specific prior written permission.\n" - "//\n" - "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" - "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" - "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" - "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" - "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" - "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" - "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" - "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" - "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" - "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" - "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" - "\n" - "/* This code will be inserted into generated code for\n" - " * google/protobuf/struct.proto. */\n" - "\n" - "/**\n" - " * Typedef representing plain JavaScript values that can go into a\n" - " * Struct.\n" - " * @typedef {null|number|string|boolean|Array|Object}\n" - " */\n" - "proto.google.protobuf.JavaScriptValue;\n" - "\n" - "\n" - "/**\n" - " * Converts this Value object to a plain JavaScript value.\n" - " * @return {?proto.google.protobuf.JavaScriptValue} a plain JavaScript\n" - " * value representing this Struct.\n" - " */\n" - "proto.google.protobuf.Value.prototype.toJavaScript = function() {\n" - " var kindCase = proto.google.protobuf.Value.KindCase;\n" - " switch (this.getKindCase()) {\n" - " case kindCase.NULL_VALUE:\n" - " return null;\n" - " case kindCase.NUMBER_VALUE:\n" - " return this.getNumberValue();\n" - " case kindCase.STRING_VALUE:\n" - " return this.getStringValue();\n" - " case kindCase.BOOL_VALUE:\n" - " return this.getBoolValue();\n" - " case kindCase.STRUCT_VALUE:\n" - " return this.getStructValue().toJavaScript();\n" - " case kindCase.LIST_VALUE:\n" - " return this.getListValue().toJavaScript();\n" - " default:\n" - " throw new Error('Unexpected struct type');\n" - " }\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Converts this JavaScript value to a new Value proto.\n" - " * @param {!proto.google.protobuf.JavaScriptValue} value The value to\n" - " * convert.\n" - " * @return {!proto.google.protobuf.Value} The newly constructed value.\n" - " */\n" - "proto.google.protobuf.Value.fromJavaScript = function(value) {\n" - " var ret = new proto.google.protobuf.Value();\n" - " switch (goog.typeOf(value)) {\n" - " case 'string':\n" - " ret.setStringValue(/** @type {string} */ (value));\n" - " break;\n" - " case 'number':\n" - " ret.setNumberValue(/** @type {number} */ (value));\n" - " break;\n" - " case 'boolean':\n" - " ret.setBoolValue(/** @type {boolean} */ (value));\n" - " break;\n" - " case 'null':\n" - " ret.setNullValue(proto.google.protobuf.NullValue.NULL_VALUE);\n" - " break;\n" - " case 'array':\n" - " ret.setListValue(proto.google.protobuf.ListValue.fromJavaScript(\n" - " /** @type{!Array} */ (value)));\n" - " break;\n" - " case 'object':\n" - " ret.setStructValue(proto.google.protobuf.Struct.fromJavaScript(\n" - " /** @type{!Object} */ (value)));\n" - " break;\n" - " default:\n" - " throw new Error('Unexpected struct type.');\n" - " }\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Converts this ListValue object to a plain JavaScript array.\n" - " * @return {!Array} a plain JavaScript array representing this List.\n" - " */\n" - "proto.google.protobuf.ListValue.prototype.toJavaScript = function() {\n" - " var ret = [];\n" - " var values = this.getValuesList();\n" - "\n" - " for (var i = 0; i < values.length; i++) {\n" - " ret[i] = values[i].toJavaScript();\n" - " }\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Constructs a ListValue protobuf from this plain JavaScript array.\n" - " * @param {!Array} array a plain JavaScript array\n" - " * @return {proto.google.protobuf.ListValue} a new ListValue object\n" - " */\n" - "proto.google.protobuf.ListValue.fromJavaScript = function(array) {\n" - " var ret = new proto.google.protobuf.ListValue();\n" - "\n" - " for (var i = 0; i < array.length; i++) {\n" - " ret.addValues(proto.google.protobuf.Value.fromJavaScript(array[i]));\n" - " }\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Converts this Struct object to a plain JavaScript object.\n" - " * @return {!Object} a plain\n" - " * JavaScript object representing this Struct.\n" - " */\n" - "proto.google.protobuf.Struct.prototype.toJavaScript = function() {\n" - " var ret = {};\n" - "\n" - " this.getFieldsMap().forEach(function(value, key) {\n" - " ret[key] = value.toJavaScript();\n" - " });\n" - "\n" - " return ret;\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Constructs a Struct protobuf from this plain JavaScript object.\n" - " * @param {!Object} obj a plain JavaScript object\n" - " * @return {proto.google.protobuf.Struct} a new Struct object\n" - " */\n" - "proto.google.protobuf.Struct.fromJavaScript = function(obj) {\n" - " var ret = new proto.google.protobuf.Struct();\n" - " var map = ret.getFieldsMap();\n" - "\n" - " for (var property in obj) {\n" - " var val = obj[property];\n" - " map.set(property, proto.google.protobuf.Value.fromJavaScript(val));\n" - " }\n" - "\n" - " return ret;\n" - "};\n" -}, -{"timestamp.js", - "// Protocol Buffers - Google's data interchange format\n" - "// Copyright 2008 Google Inc. All rights reserved.\n" - "// https://developers.google.com/protocol-buffers/\n" - "//\n" - "// Redistribution and use in source and binary forms, with or without\n" - "// modification, are permitted provided that the following conditions are\n" - "// met:\n" - "//\n" - "// * Redistributions of source code must retain the above copyright\n" - "// notice, this list of conditions and the following disclaimer.\n" - "// * Redistributions in binary form must reproduce the above\n" - "// copyright notice, this list of conditions and the following disclaimer\n" - "// in the documentation and/or other materials provided with the\n" - "// distribution.\n" - "// * Neither the name of Google Inc. nor the names of its\n" - "// contributors may be used to endorse or promote products derived from\n" - "// this software without specific prior written permission.\n" - "//\n" - "// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS\n" - "// \"AS IS\" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT\n" - "// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR\n" - "// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT\n" - "// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,\n" - "// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT\n" - "// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,\n" - "// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY\n" - "// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n" - "// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE\n" - "// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n" - "\n" - "/* This code will be inserted into generated code for\n" - " * google/protobuf/timestamp.proto. */\n" - "\n" - "/**\n" - " * Returns a JavaScript 'Date' object corresponding to this Timestamp.\n" - " * @return {!Date}\n" - " */\n" - "proto.google.protobuf.Timestamp.prototype.toDate = function() {\n" - " var seconds = this.getSeconds();\n" - " var nanos = this.getNanos();\n" - "\n" - " return new Date((seconds * 1000) + (nanos / 1000000));\n" - "};\n" - "\n" - "\n" - "/**\n" - " * Sets the value of this Timestamp object to be the given Date.\n" - " * @param {!Date} value The value to set.\n" - " */\n" - "proto.google.protobuf.Timestamp.prototype.fromDate = function(value) {\n" - " var millis = value.getTime();\n" - " this.setSeconds(Math.floor(value.getTime() / 1000));\n" - " this.setNanos(value.getMilliseconds() * 1000000);\n" - "};\n" -}, - {NULL, NULL} // Terminate the list. -}; diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index a0e1419ac12..7d10db0329c 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -14,10 +14,10 @@ # limitations under the License. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="b5fbb742af122b565925987e65c08957739976a7" +PROTOBUF_SUBMODULE_VERSION="48cb18e5c419ddd23d9badcfe4e9df7bde1979b2" diff --git a/tools/distrib/python/grpcio_tools/setup.py b/tools/distrib/python/grpcio_tools/setup.py index c6bcee497f0..c13dfe9ade5 100644 --- a/tools/distrib/python/grpcio_tools/setup.py +++ b/tools/distrib/python/grpcio_tools/setup.py @@ -160,29 +160,7 @@ def extension_modules(): plugin_sources += [ os.path.join('grpc_tools', 'main.cc'), os.path.join('grpc_root', 'src', 'compiler', 'python_generator.cc') - ] - - #HACK: Substitute the embed.cc, which is a JS to C++ - # preprocessor with the generated code. - # The generated code should not be material - # to the parts of protoc we use (it affects - # the JavaScript code generator, supposedly), - # but we need to be cautious about it. - cc_files_clone = list(CC_FILES) - embed_cc_file = os.path.normpath('google/protobuf/compiler/js/embed.cc') - well_known_types_file = os.path.normpath( - 'google/protobuf/compiler/js/well_known_types_embed.cc') - if embed_cc_file in cc_files_clone: - cc_files_clone.remove(embed_cc_file) - if well_known_types_file in cc_files_clone: - cc_files_clone.remove(well_known_types_file) - plugin_sources += [ - os.path.join('grpc_tools', - 'protobuf_generated_well_known_types_embed.cc') - ] - plugin_sources += [ - os.path.join(CC_INCLUDE, cc_file) for cc_file in cc_files_clone - ] + ] + [os.path.join(CC_INCLUDE, cc_file) for cc_file in CC_FILES] plugin_ext = extension.Extension( name='grpc_tools._protoc_compiler', diff --git a/tools/dockerfile/OWNERS b/tools/dockerfile/OWNERS index db4ab546a6d..5f0ad58d617 100644 --- a/tools/dockerfile/OWNERS +++ b/tools/dockerfile/OWNERS @@ -7,5 +7,5 @@ set noparent # for kokoro to be able to access the pre-built images. @jtattermusch -@matt-kwong +@mehrdada @nicolasnoble diff --git a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile index 3e1faafdc04..e32b3cb5e25 100644 --- a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile @@ -22,3 +22,6 @@ RUN yum install -y mono-devel RUN yum install -y nuget RUN yum install -y unzip + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index 03fb7a53438..e95d781dfd1 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index f2fa61a6916..aec936a5b8c 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile index 3edc31e1703..61ca1a08a46 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile @@ -38,3 +38,7 @@ RUN mkdir warmup \ && dotnet new \ && cd .. \ && rm -rf warmup + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 1a58f9784bf..93ee75cfcd4 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 33e2f2970c3..44fde4f99f0 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -30,24 +30,34 @@ RUN yum install -y git \ glibc-devel \ glibc-devel.i686 -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d -RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' -RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" -RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +# Install GCC 4.8 +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo # We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by # docker issue when using overlay storage driver, but all the stuff we need # will be installed, so for now we just ignore the error. # https://github.com/docker/docker/issues/10180 -RUN yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 || true +RUN yum install -y devtoolset-2-build \ + devtoolset-2-toolchain \ + devtoolset-2-binutils \ + devtoolset-2-gcc \ + devtoolset-2-gcc-c++ \ + devtoolset-2-libstdc++-devel \ + devtoolset-2-libstdc++-devel.i686 || true + +# Again, ignore the "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error. +RUN yum install -y ca-certificates || true # renew certs to prevent download error for ius-release.rpm + +# TODO(jtattermusch): gRPC makefile uses "which" to detect the availability of gcc +RUN yum install -y which || true # Update Git to version >1.7 to allow cloning submodules with --reference arg. RUN yum remove -y git && yum clean all RUN yum install -y https://centos6.iuscommunity.org/ius-release.rpm && yum clean all RUN yum install -y git2u && yum clean all -# Start in devtoolset environment that uses GCC 4.7 -CMD ["scl", "enable", "devtoolset-1.1", "bash"] +# Start in devtoolset environment that uses GCC 4.8 +CMD ["scl", "enable", "devtoolset-2", "bash"] diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 2f06bda0168..688c271feac 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index a46ebe61976..43ebf8cad95 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ @@ -1057,6 +1058,7 @@ src/core/lib/gprpp/fork.h \ src/core/lib/gprpp/inlined_vector.h \ src/core/lib/gprpp/manual_constructor.h \ src/core/lib/gprpp/memory.h \ +src/core/lib/gprpp/mutex_lock.h \ src/core/lib/gprpp/orphanable.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 4899eee3ea4..aa75bc68283 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 576950934ec..a08ce0b9ffb 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ @@ -867,6 +868,7 @@ include/grpc/support/time.h \ include/grpc/support/workaround_list.h \ src/core/README.md \ src/core/ext/README.md \ +src/core/ext/filters/census/grpc_context.cc \ src/core/ext/filters/client_channel/README.md \ src/core/ext/filters/client_channel/backup_poller.cc \ src/core/ext/filters/client_channel/backup_poller.h \ @@ -924,9 +926,12 @@ src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h \ src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc \ +src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc \ src/core/ext/filters/client_channel/resolver/dns/native/README.md \ src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc \ src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc \ @@ -1138,6 +1143,7 @@ src/core/lib/gprpp/fork.h \ src/core/lib/gprpp/inlined_vector.h \ src/core/lib/gprpp/manual_constructor.h \ src/core/lib/gprpp/memory.h \ +src/core/lib/gprpp/mutex_lock.h \ src/core/lib/gprpp/orphanable.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ @@ -1352,6 +1358,10 @@ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ src/core/lib/security/security_connector/alts_security_connector.cc \ src/core/lib/security/security_connector/alts_security_connector.h \ +src/core/lib/security/security_connector/load_system_roots.h \ +src/core/lib/security/security_connector/load_system_roots_fallback.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.h \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/local_security_connector.h \ src/core/lib/security/security_connector/security_connector.cc \ @@ -1513,7 +1523,6 @@ src/core/tsi/transport_security.h \ src/core/tsi/transport_security_grpc.cc \ src/core/tsi/transport_security_grpc.h \ src/core/tsi/transport_security_interface.h \ -src/cpp/ext/filters/census/grpc_context.cc \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ third_party/nanopb/pb_common.h \ diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc index b0feeef363c..43bc9609c7e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -17,17 +17,6 @@ # builds. This rc script must be used in the root directory of gRPC # and is expected to be used before prepare_build_macos_rc -export CONFIG=opt - -# Move gRPC repo to directory that Docker for Mac has drive access to -mkdir /Users/kbuilder/workspace -cp -R ./ /Users/kbuilder/workspace/grpc -cd /Users/kbuilder/workspace/grpc - -# Needed for identifying Docker image sha1 -brew update -brew install md5sha1sum - # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index bc7fff1b144..3212e80854e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -89,3 +89,10 @@ export DOTNET_CLI_TELEMETRY_OPTOUT=true date git submodule update --init + +# Store intermediate build files of ObjC tests into /tmpfs +mkdir /tmpfs/Build-ios-binary-size +ln -s /tmpfs/Build-ios-binary-size src/objective-c/examples/Sample/Build +mkdir /tmpfs/DerivedData +rm -rf ~/Library/Developer/Xcode/DerivedData +ln -s /tmpfs/DerivedData ~/Library/Developer/Xcode/DerivedData diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 0164e4a5617..f987f8a8cb5 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -14,7 +14,7 @@ @rem make sure msys binaries are preferred over cygwin binaries @rem set path to python 2.7 -set PATH=C:\tools\msys64\usr\bin;C:\Python27;C:\Python37;%PATH% +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( @@ -34,9 +34,6 @@ netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3 @rem Needed for big_query_utils python -m pip install google-api-python-client -@rem Install Python 3.7 -chocolatey install -y -r python3 --version 3.7 - @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 2220145dacc..42c7f5fb042 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -25,13 +25,6 @@ git submodule update --init # Build protoc and grpc_cpp_plugin. Codegen is not cross-compiled to Android make HAS_SYSTEM_PROTOBUF=false -# TODO(ericgribkoff) Remove when this commit (already in master) is included in -# next protobuf release -cd third_party/protobuf -git fetch -git cherry-pick 7daa320065f3bea2b54bf983337d1724f153422d -m 1 - - # Build and run interop instrumentation tests on Firebase Test Lab cd "${REPO_ROOT}/src/android/test/interop/" diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 5099fa03016..a6367ad9628 100644 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -14,6 +14,6 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 192d9d1b2d0..51cb66f5b86 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 6fb3c77892e..cbba9067ad3 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_full_performance_master.sh b/tools/internal_ci/linux/grpc_full_performance_master.sh index 4eddc187314..24ee71edd1b 100755 --- a/tools/internal_ci/linux/grpc_full_performance_master.sh +++ b/tools/internal_ci/linux/grpc_full_performance_master.sh @@ -21,7 +21,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_perf_multilang_rc # run 8core client vs 8core server tools/run_tests/run_performance_tests.py \ - -l c++ csharp ruby java python go php7 php7_protobuf_c \ + -l c++ csharp ruby java python go php7 php7_protobuf_c node node_purejs \ --netperf \ --category scalable \ --remote_worker_host grpc-kokoro-performance-server-8core grpc-kokoro-performance-client-8core grpc-kokoro-performance-client2-8core \ diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh old mode 100644 new mode 100755 index fecb9a5e09b..14492301cc9 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -17,6 +17,10 @@ set -ex shopt -s nullglob +cd "$(dirname "$0")/../../.." + +GRPC_VERSION=$(grep -e "^ *version: " build.yaml | head -n 1 | sed 's/.*: //') + INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts INDEX_FILENAME=index.xml @@ -24,7 +28,7 @@ BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)} BUILD_BRANCH_NAME=master BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown} BUILD_TIMESTAMP=$(date -Iseconds) -BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_ID/ +BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_GIT_COMMIT-$BUILD_ID/ GCS_ROOT=gs://packages.grpc.io/ GCS_ARCHIVE_PREFIX=archive/ @@ -43,17 +47,17 @@ find "$INPUT_ARTIFACTS" -type f PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) for zip_dir in protoc_windows_{x86,x64} do - zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$zip_dir.zip" "$INPUT_ARTIFACTS/$zip_dir/"* + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$zip_dir-$GRPC_VERSION.zip" "$INPUT_ARTIFACTS/$zip_dir/"* done for tar_dir in protoc_{linux,macos}_{x86,x64} do chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* - tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$tar_dir.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/grpc-$tar_dir-$GRPC_VERSION.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . done PROTOC_PACKAGES=( - "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_windows_{x86,x64}.zip - "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_{linux,macos}_{x86,x64}.tar.gz + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_windows_{x86,x64}-"$GRPC_VERSION.zip" + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/grpc-protoc_{linux,macos}_{x86,x64}-"$GRPC_VERSION.tar.gz" ) # C# @@ -61,6 +65,7 @@ UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" CSHARP_PACKAGES=( "$UNZIPPED_CSHARP_PACKAGES"/* + "$INPUT_ARTIFACTS"/grpc_unity_package.[0-9]*.zip ) # Python diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 25531ead2b5..2ba7d469ecf 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index d9b039f1ecd..338b1b6a0d0 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -61,8 +61,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --runs_per_test_detects_flakes \ - --runs_per_test=2 \ + --cache_test_results=no \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh index 5ddabb9bf9c..e748a62e761 100755 --- a/tools/internal_ci/macos/grpc_interop_toprod.sh +++ b/tools/internal_ci/macos/grpc_interop_toprod.sh @@ -18,8 +18,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc source tools/internal_ci/helper_scripts/prepare_build_macos_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc # using run_interop_tests.py without --use_docker, so we need to build first tools/run_tests/run_tests.py -l c++ -c opt --build_only diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 4d528e01285..0e02bdb6ecc 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -18,6 +18,7 @@ rename C:\Python27_32bit Python27_32bits rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits +rename C:\Python37_32bit Python37_32bits @rem enter repo root cd /d %~dp0\..\..\.. diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 40c02a1bc7a..db84d9b454c 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -9,7 +9,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are: - Add (or update) an entry in `./client_matrix.py` file to reference the github tag for the release. - Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do - - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language cxx csharp python ruby php` + - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php` - Verify that the new docker image was built successfully and uploaded to GCR. For example, - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index fae78909daf..bb9222d953f 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -96,6 +96,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'go': [ { @@ -178,6 +181,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.1': None }, + { + 'v1.14.0': None + }, ], 'python': [ { @@ -219,6 +225,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'node': [ { @@ -304,6 +313,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'php': [ { @@ -345,6 +357,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'csharp': [ { @@ -391,6 +406,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], } diff --git a/tools/profiling/ios_bin/binary_size.py b/tools/profiling/ios_bin/binary_size.py index b07adb57345..d4d134fef37 100755 --- a/tools/profiling/ios_bin/binary_size.py +++ b/tools/profiling/ios_bin/binary_size.py @@ -55,7 +55,7 @@ def dir_size(dir): def get_size(where, frameworks): - build_dir = 'src/objective-c/examples/Sample/Build-%s/' % where + build_dir = 'src/objective-c/examples/Sample/Build/Build-%s/' % where if not frameworks: link_map_filename = 'Build/Intermediates.noindex/Sample.build/Release-iphoneos/Sample.build/Sample-LinkMap-normal-arm64.txt' return parse_link_map(build_dir + link_map_filename) @@ -75,15 +75,17 @@ def get_size(where, frameworks): def build(where, frameworks): + subprocess.check_call(['make', 'clean']) shutil.rmtree( - 'src/objective-c/examples/Sample/Build-%s' % where, ignore_errors=True) + 'src/objective-c/examples/Sample/Build/Build-%s' % where, + ignore_errors=True) subprocess.check_call( 'CONFIG=opt EXAMPLE_PATH=src/objective-c/examples/Sample SCHEME=Sample FRAMEWORKS=%s ./build_one_example.sh' % ('YES' if frameworks else 'NO'), shell=True, cwd='src/objective-c/tests') - os.rename('src/objective-c/examples/Sample/Build', - 'src/objective-c/examples/Sample/Build-%s' % where) + os.rename('src/objective-c/examples/Sample/Build/Build', + 'src/objective-c/examples/Sample/Build/Build-%s' % where) text = 'Objective-C binary sizes\n' diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 4500b220e97..bdeb258e1fc 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -234,6 +234,11 @@ class CSharpExtArtifact: environ={ 'ANDROID_ABI': self.arch_abi }) + elif self.arch == 'ios': + return create_jobspec( + self.name, + ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], + use_workspace=True) elif self.platform == 'windows': cmake_arch_option = 'Win32' if self.arch == 'x86' else self.arch return create_jobspec( @@ -285,15 +290,9 @@ class PHPArtifact: return [] def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec( - self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( - self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') - else: - return create_jobspec( - self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], - use_workspace=True) + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( + self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') class ProtocArtifact: @@ -356,6 +355,8 @@ def targets(): ] + [ CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), + CSharpExtArtifact('linux', 'android', arch_abi='x86'), + CSharpExtArtifact('macos', 'ios'), PythonArtifact('linux', 'x86', 'cp27-cp27m'), PythonArtifact('linux', 'x86', 'cp27-cp27mu'), PythonArtifact('linux', 'x86', 'cp34-cp34m'), @@ -385,6 +386,7 @@ def targets(): PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), + PythonArtifact('windows', 'x86', 'Python37_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), @@ -392,6 +394,5 @@ def targets(): PythonArtifact('windows', 'x64', 'Python37'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64'), - PHPArtifact('macos', 'x64') + PHPArtifact('linux', 'x64') ]) diff --git a/tools/run_tests/artifacts/build_artifact_csharp_ios.sh b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh new file mode 100755 index 00000000000..c902a45db7d --- /dev/null +++ b/tools/run_tests/artifacts/build_artifact_csharp_ios.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +cd "$(dirname "$0")/../../.." + +src/csharp/experimental/build_native_ext_for_ios.sh + +mkdir -p "${ARTIFACTS_OUT}" +cp libs/ios/libgrpc_csharp_ext.a libs/ios/libgrpc.a "${ARTIFACTS_OUT}" diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index b531fc9a5d4..6d433f2dad4 100755 --- a/tools/run_tests/artifacts/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use devtoolset environment that has GCC 4.7 before set -ex -source scl_source enable devtoolset-1.1 +# Use devtoolset environment that has GCC 4.8 before set -ex +source scl_source enable devtoolset-2 set -ex diff --git a/tools/run_tests/artifacts/build_package_php.sh b/tools/run_tests/artifacts/build_package_php.sh index 9a8f25a6f87..e263e09a357 100755 --- a/tools/run_tests/artifacts/build_package_php.sh +++ b/tools/run_tests/artifacts/build_package_php.sh @@ -20,7 +20,4 @@ cd "$(dirname "$0")/../../.." # All the PHP packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. mkdir -p artifacts/ -# Jenkins flow (deprecated) -cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/php_*/* artifacts/ || true -# Kokoro flow cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/php_*/* artifacts/ || true diff --git a/tools/run_tests/artifacts/build_package_python.sh b/tools/run_tests/artifacts/build_package_python.sh index 15627881e8a..29801a5b867 100755 --- a/tools/run_tests/artifacts/build_package_python.sh +++ b/tools/run_tests/artifacts/build_package_python.sh @@ -21,9 +21,6 @@ mkdir -p artifacts/ # All the python packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -# Jenkins flow (deprecated) -cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/python_*/* artifacts/ || true -# Kokoro flow cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/python_*/* artifacts/ || true # TODO: all the artifact builder configurations generate a grpcio-VERSION.tar.gz diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index 4f74f082760..bd62ec1ff46 100755 --- a/tools/run_tests/artifacts/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -23,9 +23,6 @@ mkdir -p artifacts/ # All the ruby packages have been built in the artifact phase already # and we only collect them here to deliver them to the distribtest phase. -# Jenkins flow (deprecated) -cp -r "${EXTERNAL_GIT_ROOT}"/platform={windows,linux,macos}/artifacts/ruby_native_gem_*/* artifacts/ || true -# Kokoro flow cp -r "${EXTERNAL_GIT_ROOT}"/input_artifacts/ruby_native_gem_*/* artifacts/ || true well_known_protos=( any api compiler/plugin descriptor duration empty field_mask source_context struct timestamp type wrappers ) @@ -44,12 +41,7 @@ for arch in {x86,x64}; do ;; esac for plat in {windows,linux,macos}; do - if [ "${KOKORO_JOB_NAME}" != "" ] - then - input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}" - else - input_dir="${EXTERNAL_GIT_ROOT}/platform=${plat}/artifacts/protoc_${plat}_${arch}" - fi + input_dir="${EXTERNAL_GIT_ROOT}/input_artifacts/protoc_${plat}_${arch}" output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" mkdir -p "$output_dir"/google/protobuf mkdir -p "$output_dir"/google/protobuf/compiler # needed for plugin.proto diff --git a/tools/run_tests/artifacts/package_targets.py b/tools/run_tests/artifacts/package_targets.py index abf1b5ebed2..70f1fc8a4b5 100644 --- a/tools/run_tests/artifacts/package_targets.py +++ b/tools/run_tests/artifacts/package_targets.py @@ -56,7 +56,8 @@ def create_jobspec(name, cwd=None, shell=False, flake_retries=0, - timeout_retries=0): + timeout_retries=0, + cpu_cost=1.0): """Creates jobspec.""" jobspec = jobset.JobSpec( cmdline=cmdline, @@ -66,31 +67,36 @@ def create_jobspec(name, timeout_seconds=10 * 60, flake_retries=flake_retries, timeout_retries=timeout_retries, + cpu_cost=cpu_cost, shell=shell) return jobspec class CSharpPackage: - """Builds C# nuget packages.""" - - def __init__(self, linux=False): - self.linux = linux - self.labels = ['package', 'csharp'] - if linux: - self.name = 'csharp_package_dotnetcli_linux' - self.labels += ['linux'] + """Builds C# packages.""" + + def __init__(self, unity=False): + self.unity = unity + self.labels = ['package', 'csharp', 'windows'] + if unity: + self.name = 'csharp_package_unity_windows' + self.labels += ['unity'] else: - self.name = 'csharp_package_dotnetcli_windows' - self.labels += ['windows'] + self.name = 'csharp_package_nuget_windows' + self.labels += ['nuget'] def pre_build_jobspecs(self): return [] def build_jobspec(self): - if self.linux: - return create_docker_jobspec( - self.name, 'tools/dockerfile/test/csharp_jessie_x64', - 'src/csharp/build_packages_dotnetcli.sh') + if self.unity: + # use very high CPU cost to avoid running nuget package build + # and unity build concurrently + return create_jobspec( + self.name, ['build_unitypackage.bat'], + cwd='src\\csharp', + cpu_cost=1e6, + shell=True) else: return create_jobspec( self.name, ['build_packages_dotnetcli.bat'], @@ -153,7 +159,7 @@ def targets(): """Gets list of supported targets""" return [ CSharpPackage(), - CSharpPackage(linux=True), + CSharpPackage(unity=True), RubyPackage(), PythonPackage(), PHPPackage() diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 4ef74085f94..3f01fbc7b7d 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -73,6 +73,10 @@ docker run \ # Copy output artifacts if [ "$OUTPUT_DIR" != "" ] then + # Create the artifact directory in advance to avoid a race in "docker cp" if tasks + # that were running in parallel finish at the same time. + # see https://github.com/grpc/grpc/issues/16155 + mkdir -p "$git_root/$OUTPUT_DIR" docker cp "$CONTAINER_NAME:/var/local/git/grpc/$OUTPUT_DIR" "$git_root" || FAILED="true" fi diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 072402b2cf6..bff045f7860 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -1032,6 +1032,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "grpc_ipv6_loopback_available_test", + "src": [ + "test/core/iomgr/grpc_ipv6_loopback_available_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -3690,6 +3707,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "src": [ + "test/core/security/linux_system_roots_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "grpc_plugin_support" @@ -9153,7 +9187,7 @@ "name": "census", "src": [ "include/grpc/census.h", - "src/cpp/ext/filters/census/grpc_context.cc" + "src/core/ext/filters/census/grpc_context.cc" ], "third_party": false, "type": "filegroup" @@ -9272,6 +9306,7 @@ "src/core/lib/gprpp/fork.h", "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/memory.h", + "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h" ], @@ -9319,6 +9354,7 @@ "src/core/lib/gprpp/fork.h", "src/core/lib/gprpp/manual_constructor.h", "src/core/lib/gprpp/memory.h", + "src/core/lib/gprpp/mutex_lock.h", "src/core/lib/gprpp/thd.h", "src/core/lib/profiling/timers.h" ], @@ -9887,6 +9923,7 @@ "src/core/lib/iomgr/endpoint_cfstream.h", "src/core/lib/iomgr/error_cfstream.cc", "src/core/lib/iomgr/error_cfstream.h", + "src/core/lib/iomgr/iomgr_posix_cfstream.cc", "src/core/lib/iomgr/tcp_client_cfstream.cc" ], "third_party": false, @@ -10254,9 +10291,12 @@ "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc", "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h", - "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc" + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc", + "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc" ], "third_party": false, "type": "filegroup" @@ -10338,6 +10378,8 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", @@ -10386,6 +10428,10 @@ "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.cc", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.cc", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index a5439a5db13..a51be28ad51 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -1313,6 +1313,32 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [ + "uv" + ], + "flaky": false, + "gtest": false, + "language": "c", + "name": "grpc_ipv6_loopback_available_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -3457,7 +3483,7 @@ "mac", "posix" ], - "uses_polling": true + "uses_polling": false }, { "args": [], @@ -4307,6 +4333,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -5710,7 +5760,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5722,7 +5773,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, @@ -5732,7 +5784,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5744,7 +5797,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, @@ -5754,7 +5808,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5766,7 +5821,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, diff --git a/tools/run_tests/performance/OWNERS b/tools/run_tests/performance/OWNERS index fc1d6eec9c7..9cf8c131111 100644 --- a/tools/run_tests/performance/OWNERS +++ b/tools/run_tests/performance/OWNERS @@ -5,5 +5,5 @@ set noparent # to update the BigQuery schema @ncteisen -@matt-kwong +@apolcyn @jtattermusch diff --git a/tools/run_tests/performance/README.md b/tools/run_tests/performance/README.md index 2fc1a27c9bc..791270ab389 100644 --- a/tools/run_tests/performance/README.md +++ b/tools/run_tests/performance/README.md @@ -104,3 +104,31 @@ Example memory profile of grpc-go server, with `go tools pprof`: ``` $ go tool pprof --text --alloc_space http://localhost:/debug/heap ``` + +### Configuration environment variables: + +* QPS_WORKER_CHANNEL_CONNECT_TIMEOUT + + Consuming process: qps_worker + + Type: integer (number of seconds) + + This can be used to configure the amount of time that benchmark + clients wait for channels to the benchmark server to become ready. + This is useful in certain benchmark environments in which the + server can take a long time to become ready. Note: if setting + this to a high value, then the scenario config under test should + probably also have a large "warmup_seconds". + +* QPS_WORKERS + + Consuming process: qps_json_driver + + Type: comma separated list of host:port + + Set this to a comma separated list of QPS worker processes/machines. + Each scenario in a scenario config has specifies a certain number + of servers, `num_servers`, and the driver will start + "benchmark servers"'s on the first `num_server` `host:port` pairs in + the comma separated list. The rest will be told to run as clients + against the benchmark server. diff --git a/tools/run_tests/performance/bq_upload_result.py b/tools/run_tests/performance/bq_upload_result.py index 67025875578..b442f0cf83a 100755 --- a/tools/run_tests/performance/bq_upload_result.py +++ b/tools/run_tests/performance/bq_upload_result.py @@ -128,14 +128,16 @@ def _flatten_result_inplace(scenario_result): def _populate_metadata_inplace(scenario_result): """Populates metadata based on environment variables set by Jenkins.""" - # NOTE: Grabbing the Jenkins environment variables will only work if the - # driver is running locally on the same machine where Jenkins has started + # NOTE: Grabbing the Kokoro environment variables will only work if the + # driver is running locally on the same machine where Kokoro has started # the job. For our setup, this is currently the case, so just assume that. - build_number = os.getenv('BUILD_NUMBER') - build_url = os.getenv('BUILD_URL') - job_name = os.getenv('JOB_NAME') - git_commit = os.getenv('GIT_COMMIT') + build_number = os.getenv('KOKORO_BUILD_NUMBER') + build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( + 'KOKORO_BUILD_ID') + job_name = os.getenv('KOKORO_JOB_NAME') + git_commit = os.getenv('KOKORO_GIT_COMMIT') # actual commit is the actual head of PR that is getting tested + # TODO(jtattermusch): unclear how to obtain on Kokoro git_actual_commit = os.getenv('ghprbActualCommit') utc_timestamp = str(calendar.timegm(time.gmtime())) diff --git a/tools/run_tests/performance/build_performance.sh b/tools/run_tests/performance/build_performance.sh index 22e0ca9fa07..35d9e905986 100755 --- a/tools/run_tests/performance/build_performance.sh +++ b/tools/run_tests/performance/build_performance.sh @@ -55,6 +55,9 @@ do "csharp") python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 --compiler coreclr ;; + "node"|"node_purejs") + tools/run_tests/performance/build_performance_node.sh + ;; *) python tools/run_tests/run_tests.py -l "$language" -c "$CONFIG" --build_only -j 8 ;; diff --git a/tools/run_tests/performance/build_performance_node.sh b/tools/run_tests/performance/build_performance_node.sh new file mode 100755 index 00000000000..12e08722648 --- /dev/null +++ b/tools/run_tests/performance/build_performance_node.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set +ex + +. "$HOME/.nvm/nvm.sh" + +nvm install 10 + +set -ex + +cd "$(dirname "$0")/../../../../grpc-node" + +npm install + +./node_modules/.bin/gulp setup diff --git a/tools/run_tests/performance/run_worker_node.sh b/tools/run_tests/performance/run_worker_node.sh new file mode 100755 index 00000000000..3e5dd18edb3 --- /dev/null +++ b/tools/run_tests/performance/run_worker_node.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +. "$HOME/.nvm/nvm.sh" + +nvm use 10 + +set -ex + +fixture=$1 + +shift + +# Enter repo root +cd "$(dirname "$0")/../../.." + +# Enter the grpc-node repo root (expected to be next to grpc repo root) +cd ../grpc-node + +node -r "./test/fixtures/$fixture" test/performance/worker.js "$@" diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py index f05753154e9..2e78bd07fb2 100644 --- a/tools/run_tests/performance/scenario_config.py +++ b/tools/run_tests/performance/scenario_config.py @@ -1151,6 +1151,106 @@ class GoLanguage: return 'go' +class NodeLanguage: + + def __init__(self, node_purejs=False): + pass + self.node_purejs = node_purejs + self.safename = str(self) + + def worker_cmdline(self): + fixture = 'native_js' if self.node_purejs else 'native_native' + return [ + 'tools/run_tests/performance/run_worker_node.sh', fixture, + '--benchmark_impl=grpc' + ] + + def worker_port_offset(self): + if self.node_purejs: + return 1100 + return 1000 + + def scenarios(self): + node_implementation = 'node_purejs' if self.node_purejs else 'node' + for secure in [True, False]: + secstr = 'secure' if secure else 'insecure' + smoketest_categories = ([SMOKETEST] if secure else []) + [SCALABLE] + + yield _ping_pong_scenario( + '%s_to_node_generic_async_streaming_ping_pong_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + server_language='node', + use_generic_payload=True, + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_streaming_ping_pong_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + async_server_threads=1, + secure=secure) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_unary_ping_pong_%s' % + (node_implementation, secstr), + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + async_server_threads=1, + secure=secure, + categories=smoketest_categories) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_unary_qps_unconstrained_%s' % + (node_implementation, secstr), + rpc_type='UNARY', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + unconstrained_client='async', + secure=secure, + categories=smoketest_categories + [SCALABLE]) + + yield _ping_pong_scenario( + '%s_to_node_protobuf_async_streaming_qps_unconstrained_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_SERVER', + server_language='node', + unconstrained_client='async', + secure=secure, + categories=[SCALABLE]) + + yield _ping_pong_scenario( + '%s_to_node_generic_async_streaming_qps_unconstrained_%s' % + (node_implementation, secstr), + rpc_type='STREAMING', + client_type='ASYNC_CLIENT', + server_type='ASYNC_GENERIC_SERVER', + server_language='node', + unconstrained_client='async', + use_generic_payload=True, + secure=secure, + categories=[SCALABLE]) + + # TODO(murgatroid99): add scenarios node vs C++ + + def __str__(self): + if self.node_purejs: + return 'node_purejs' + return 'node' + + LANGUAGES = { 'c++': CXXLanguage(), 'csharp': CSharpLanguage(), @@ -1160,4 +1260,6 @@ LANGUAGES = { 'java': JavaLanguage(), 'python': PythonLanguage(), 'go': GoLanguage(), + 'node': NodeLanguage(), + 'node_purejs': NodeLanguage(node_purejs=True) } diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index cbb4c32a2af..9d997037259 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -68,15 +68,13 @@ _INTEROP_RESULTS_SCHEMA = [ def _get_build_metadata(test_results): - """Add Jenkins/Kokoro build metadata to test_results based on environment - variables set by Jenkins/Kokoro. + """Add Kokoro build metadata to test_results based on environment + variables set by Kokoro. """ - build_id = os.getenv('BUILD_ID') or os.getenv('KOKORO_BUILD_NUMBER') - build_url = os.getenv('BUILD_URL') - if os.getenv('KOKORO_BUILD_ID'): - build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( - 'KOKORO_BUILD_ID') - job_name = os.getenv('JOB_BASE_NAME') or os.getenv('KOKORO_JOB_NAME') + build_id = os.getenv('KOKORO_BUILD_NUMBER') + build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( + 'KOKORO_BUILD_ID') + job_name = os.getenv('KOKORO_JOB_NAME') if build_id: test_results['build_id'] = build_id diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index aa58107ced9..22055d58e8b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -637,13 +637,13 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 'java', 'go', 'python', 'c++' ] -#TODO: Add c++ when c++ ALTS interop client is ready. _LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -#TODO: Add c++ when c++ ALTS interop server is ready. _SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] +_TRANSPORT_SECURITY_OPTIONS = [ + 'tls', 'alts', 'google_default_credentials', 'insecure' +] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -724,6 +724,9 @@ def auth_options(language, test_case, service_account_key_file=None): key_file_arg = '--service_account_key_file=%s' % service_account_key_file default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' + # TODO: When using google_default_credentials outside of cloud-to-prod, the environment variable + # 'GOOGLE_APPLICATION_CREDENTIALS' needs to be set for the test case + # 'jwt_token_creds' to work. if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: if language in [ 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', @@ -763,15 +766,25 @@ def cloud_to_prod_jobspec(language, docker_image=None, auth=False, manual_cmd_log=None, - service_account_key_file=None): + service_account_key_file=None, + transport_security='tls'): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ '--server_host=%s' % server_host, '--server_host_override=%s' % server_host, '--server_port=443', - '--use_tls=true', '--test_case=%s' % test_case ] + if transport_security == 'tls': + transport_security_options += ['--use_tls=true'] + elif transport_security == 'google_default_credentials' and language == 'c++': + transport_security_options += [ + '--custom_credentials_type=google_default_credentials' + ] + else: + print('Invalid transport security option.') + sys.exit(1) + cmdargs = cmdargs + transport_security_options environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: auth_cmdargs, auth_env = auth_options(language, test_case, @@ -1285,14 +1298,16 @@ try: jobs = [] if args.cloud_to_prod: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1300,8 +1315,23 @@ try: docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: @@ -1312,12 +1342,15 @@ try: prod_servers[server_host_nickname], docker_image=docker_images.get(str(http2Interop)), manual_cmd_log=client_manual_cmd_log, - service_account_key_file=args.service_account_key_file) + service_account_key_file=args.service_account_key_file, + transport_security=args.transport_security) jobs.append(test_job) if args.cloud_to_prod_auth: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _AUTH_TEST_CASES: @@ -1325,7 +1358,7 @@ try: not compute_engine_creds_required( language, test_case)): if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1334,8 +1367,23 @@ try: auth=True, manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) for server in args.override_server: server_name = server[0] diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py index 9a9f74e9e5a..04e706fa5b9 100755 --- a/tools/run_tests/run_performance_tests.py +++ b/tools/run_tests/run_performance_tests.py @@ -189,11 +189,19 @@ def create_netperf_jobspec(server_host='localhost', def archive_repo(languages): """Archives local version of repo including submodules.""" - cmdline = ['tar', '-cf', '../grpc.tar', '../grpc/'] + # Directory contains symlinks that can't be correctly untarred on Windows + # so we just skip them as a workaround. + # See https://github.com/grpc/grpc/issues/16334 + bad_symlinks_dir = '../grpc/third_party/libcxx/test/std/experimental/filesystem/Inputs/static_test_env' + cmdline = [ + 'tar', '--exclude', bad_symlinks_dir, '-cf', '../grpc.tar', '../grpc/' + ] if 'java' in languages: cmdline.append('../grpc-java') if 'go' in languages: cmdline.append('../grpc-go') + if 'node' in languages or 'node_purejs' in languages: + cmdline.append('../grpc-node') archive_job = jobset.JobSpec( cmdline=cmdline, shortname='archive_repo', timeout_seconds=3 * 60) @@ -247,9 +255,9 @@ def build_on_remote_hosts(hosts, languages=scenario_config.LANGUAGES.keys(), build_local=False): """Builds performance worker on remote hosts (and maybe also locally).""" - build_timeout = 15 * 60 + build_timeout = 45 * 60 # Kokoro VMs (which are local only) do not have caching, so they need more time to build - local_build_timeout = 30 * 60 + local_build_timeout = 60 * 60 build_jobs = [] for host in hosts: user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, host) diff --git a/tools/run_tests/sanity/check_unsecure.sh b/tools/run_tests/sanity/check_bad_dependencies.sh similarity index 87% rename from tools/run_tests/sanity/check_unsecure.sh rename to tools/run_tests/sanity/check_bad_dependencies.sh index cca1235479e..5ae0e02c81d 100755 --- a/tools/run_tests/sanity/check_unsecure.sh +++ b/tools/run_tests/sanity/check_bad_dependencies.sh @@ -23,5 +23,9 @@ test "$(bazel query 'somepath("//:grpc++_unsecure", "//external:libssl")' 2>/dev test "$(bazel query 'somepath("//:grpc++_codegen_proto", "//external:libssl")' 2>/dev/null | wc -l)" -eq 0 || exit 1 test "$(bazel query 'somepath("//test/cpp/microbenchmarks:helpers", "//external:libssl")' 2>/dev/null | wc -l)" -eq 0 || exit 1 +# Make sure that core doesn't depend on anything in C++ library + +test "$(bazel query 'deps("//:grpc")' 2>/dev/null | egrep 'src/cpp|include/grpcpp' | wc -l)" -eq 0 || exit 1 + exit 0 diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index fb159bc2d20..2c7c140716f 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -36,7 +36,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) - b5fbb742af122b565925987e65c08957739976a7 third_party/protobuf (v3.3.1-641-gb5fbb74) + 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 third_party/protobuf (v3.6.0.1-37-g48cb18e5) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF diff --git a/tools/run_tests/sanity/sanity_tests.yaml b/tools/run_tests/sanity/sanity_tests.yaml index ac0d4c70e5d..91b53eb38d4 100644 --- a/tools/run_tests/sanity/sanity_tests.yaml +++ b/tools/run_tests/sanity/sanity_tests.yaml @@ -1,4 +1,5 @@ # a set of tests that are run in parallel for sanity tests +- script: tools/run_tests/sanity/check_bad_dependencies.sh - script: tools/run_tests/sanity/check_bazel_workspace.py - script: tools/run_tests/sanity/check_cache_mk.sh - script: tools/run_tests/sanity/check_owners.sh @@ -6,7 +7,6 @@ - script: tools/run_tests/sanity/check_submodules.sh - script: tools/run_tests/sanity/check_test_filtering.py - script: tools/run_tests/sanity/check_tracer_sanity.py -- script: tools/run_tests/sanity/check_unsecure.sh - script: tools/run_tests/sanity/core_banned_functions.py - script: tools/run_tests/sanity/core_untyped_structs.sh - script: tools/run_tests/sanity/check_deprecated_grpc++.py