diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12f46d181e6..c28eb974f88 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,7 +3,7 @@ # repository as the source of truth for module ownership. /**/OWNERS @markdroth @nicolasnoble @a11r /bazel/** @nicolasnoble @dgquintas @a11r @vjpai -/cmake/** @jtattermusch @nicolasnoble @matt-kwong +/cmake/** @jtattermusch @nicolasnoble @mehrdada /src/core/ext/filters/client_channel/** @markdroth @dgquintas @AspirinSJL -/tools/dockerfile/** @jtattermusch @matt-kwong @nicolasnoble -/tools/run_tests/performance/** @ncteisen @matt-kwong @jtattermusch +/tools/dockerfile/** @jtattermusch @mehrdada @nicolasnoble +/tools/run_tests/performance/** @ncteisen @apolcyn @jtattermusch diff --git a/.github/mergeable.yml b/.github/mergeable.yml index 660d8cb4405..30692095c46 100644 --- a/.github/mergeable.yml +++ b/.github/mergeable.yml @@ -1,6 +1,14 @@ mergeable: pull_requests: label: - must_include: - regex: "release notes: yes|release notes: no" - message: "Add release notes yes/no label. For yes, add lang label" + or: + - and: + - must_include: + regex: 'release notes: yes' + message: 'Please include release note: yes' + - must_include: + regex: '^lang\/' + message: 'Please include a language label' + - must_include: + regex: 'release notes: no' + message: 'Please include release note: no' diff --git a/BUILD b/BUILD index 81390dd1aa1..35cf86288d6 100644 --- a/BUILD +++ b/BUILD @@ -1499,6 +1499,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.cc", "src/core/lib/security/credentials/ssl/ssl_credentials.cc", "src/core/lib/security/security_connector/alts_security_connector.cc", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/security_connector.cc", "src/core/lib/security/transport/client_auth_filter.cc", @@ -1527,6 +1529,8 @@ grpc_cc_library( "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", diff --git a/CMakeLists.txt b/CMakeLists.txt index e8e65d4b71d..c4526d2af5d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,8 @@ if (MSVC) add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later @@ -579,6 +581,7 @@ add_dependencies(buildtests_cxx generic_end2end_test) add_dependencies(buildtests_cxx golden_file_test) add_dependencies(buildtests_cxx grpc_alts_credentials_options_test) add_dependencies(buildtests_cxx grpc_cli) +add_dependencies(buildtests_cxx grpc_linux_system_roots_test) add_dependencies(buildtests_cxx grpc_tool_test) add_dependencies(buildtests_cxx grpclb_api_test) add_dependencies(buildtests_cxx grpclb_end2end_test) @@ -660,12 +663,8 @@ add_dependencies(buildtests_cxx transport_security_common_api_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test_unsecure) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test) -endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) endif() @@ -674,9 +673,7 @@ add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) endif() add_dependencies(buildtests_cxx address_sorting_test_unsecure) add_dependencies(buildtests_cxx address_sorting_test) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx cancel_ares_query_test) -endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -1133,6 +1130,8 @@ add_library(grpc src/core/lib/security/credentials/plugin/plugin_credentials.cc src/core/lib/security/credentials/ssl/ssl_credentials.cc src/core/lib/security/security_connector/alts_security_connector.cc + src/core/lib/security/security_connector/load_system_roots_fallback.cc + src/core/lib/security/security_connector/load_system_roots_linux.cc src/core/lib/security/security_connector/local_security_connector.cc src/core/lib/security/security_connector/security_connector.cc src/core/lib/security/transport/client_auth_filter.cc @@ -1563,6 +1562,8 @@ add_library(grpc_cronet src/core/lib/security/credentials/plugin/plugin_credentials.cc src/core/lib/security/credentials/ssl/ssl_credentials.cc src/core/lib/security/security_connector/alts_security_connector.cc + src/core/lib/security/security_connector/load_system_roots_fallback.cc + src/core/lib/security/security_connector/load_system_roots_linux.cc src/core/lib/security/security_connector/local_security_connector.cc src/core/lib/security/security_connector/security_connector.cc src/core/lib/security/transport/client_auth_filter.cc @@ -12150,6 +12151,44 @@ if (gRPC_INSTALL) endif() endif (gRPC_BUILD_CODEGEN) +if (gRPC_BUILD_TESTS) + +add_executable(grpc_linux_system_roots_test + test/core/security/linux_system_roots_test.cc + third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc +) + + +target_include_directories(grpc_linux_system_roots_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_NANOPB_INCLUDE_DIR} + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_linux_system_roots_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_CODEGEN) add_executable(grpc_node_plugin @@ -16211,7 +16250,6 @@ target_link_libraries(inproc_nosec_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test_unsecure test/cpp/naming/resolver_component_test.cc @@ -16251,10 +16289,8 @@ target_link_libraries(resolver_component_test_unsecure ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test test/cpp/naming/resolver_component_test.cc @@ -16294,7 +16330,6 @@ target_link_libraries(resolver_component_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -16465,7 +16500,6 @@ target_link_libraries(address_sorting_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(cancel_ares_query_test test/cpp/naming/cancel_ares_query_test.cc @@ -16505,7 +16539,6 @@ target_link_libraries(cancel_ares_query_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/Makefile b/Makefile index 5174ab6719c..9d23de866cd 100644 --- a/Makefile +++ b/Makefile @@ -1170,6 +1170,7 @@ grpc_alts_credentials_options_test: $(BINDIR)/$(CONFIG)/grpc_alts_credentials_op grpc_cli: $(BINDIR)/$(CONFIG)/grpc_cli grpc_cpp_plugin: $(BINDIR)/$(CONFIG)/grpc_cpp_plugin grpc_csharp_plugin: $(BINDIR)/$(CONFIG)/grpc_csharp_plugin +grpc_linux_system_roots_test: $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test grpc_node_plugin: $(BINDIR)/$(CONFIG)/grpc_node_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin grpc_php_plugin: $(BINDIR)/$(CONFIG)/grpc_php_plugin @@ -1670,6 +1671,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ @@ -1849,6 +1851,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test \ $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/grpclb_end2end_test \ @@ -2316,6 +2319,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/golden_file_test || ( echo test golden_file_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_alts_credentials_options_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_alts_credentials_options_test || ( echo test grpc_alts_credentials_options_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_linux_system_roots_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test || ( echo test grpc_linux_system_roots_test failed ; exit 1 ) $(E) "[RUN] Testing grpc_tool_test" $(Q) $(BINDIR)/$(CONFIG)/grpc_tool_test || ( echo test grpc_tool_test failed ; exit 1 ) $(E) "[RUN] Testing grpclb_api_test" @@ -3608,6 +3613,8 @@ LIBGRPC_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -4037,6 +4044,8 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ @@ -17907,6 +17916,49 @@ ifneq ($(NO_DEPS),true) endif +GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC = \ + test/core/security/linux_system_roots_test.cc \ + +GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_LINUX_SYSTEM_ROOTS_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.5.0+. + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test: $(PROTOBUF_DEP) $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_linux_system_roots_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/security/linux_system_roots_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_linux_system_roots_test: $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_LINUX_SYSTEM_ROOTS_TEST_OBJS:.o=.dep) +endif +endif + + GRPC_NODE_PLUGIN_SRC = \ src/compiler/node_plugin.cc \ @@ -24651,6 +24703,8 @@ src/core/lib/security/credentials/oauth2/oauth2_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/credentials/plugin/plugin_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/credentials/ssl/ssl_credentials.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/alts_security_connector.cc: $(OPENSSL_DEP) +src/core/lib/security/security_connector/load_system_roots_fallback.cc: $(OPENSSL_DEP) +src/core/lib/security/security_connector/load_system_roots_linux.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/local_security_connector.cc: $(OPENSSL_DEP) src/core/lib/security/security_connector/security_connector.cc: $(OPENSSL_DEP) src/core/lib/security/transport/client_auth_filter.cc: $(OPENSSL_DEP) diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 7c349dbcdcf..d7519b14532 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -116,9 +116,9 @@ def grpc_deps(): if "com_google_protobuf" not in native.existing_rules(): native.http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-b5fbb742af122b565925987e65c08957739976a7", - url = "https://github.com/google/protobuf/archive/b5fbb742af122b565925987e65c08957739976a7.tar.gz", - ) + strip_prefix = "protobuf-48cb18e5c419ddd23d9badcfe4e9df7bde1979b2", + url = "https://github.com/google/protobuf/archive/48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz", + ) if "com_github_nanopb_nanopb" not in native.existing_rules(): native.new_http_archive( diff --git a/build.yaml b/build.yaml index 70af96046cb..bd9c4237a11 100644 --- a/build.yaml +++ b/build.yaml @@ -792,6 +792,8 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.h - src/core/lib/security/credentials/ssl/ssl_credentials.h - src/core/lib/security/security_connector/alts_security_connector.h + - src/core/lib/security/security_connector/load_system_roots.h + - src/core/lib/security/security_connector/load_system_roots_linux.h - src/core/lib/security/security_connector/local_security_connector.h - src/core/lib/security/security_connector/security_connector.h - src/core/lib/security/transport/auth_filters.h @@ -819,6 +821,8 @@ filegroups: - src/core/lib/security/credentials/plugin/plugin_credentials.cc - src/core/lib/security/credentials/ssl/ssl_credentials.cc - src/core/lib/security/security_connector/alts_security_connector.cc + - src/core/lib/security/security_connector/load_system_roots_fallback.cc + - src/core/lib/security/security_connector/load_system_roots_linux.cc - src/core/lib/security/security_connector/local_security_connector.cc - src/core/lib/security/security_connector/security_connector.cc - src/core/lib/security/transport/client_auth_filter.cc @@ -4698,6 +4702,17 @@ targets: secure: false vs_config_type: Application vs_project_guid: '{3C813052-A49A-4662-B90A-1ADBEC7EE453}' +- name: grpc_linux_system_roots_test + gtest: true + build: test + language: c++ + src: + - test/core/security/linux_system_roots_test.cc + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: grpc_node_plugin build: protoc language: c++ diff --git a/cmake/OWNERS b/cmake/OWNERS index 530a941c466..21981a7c555 100644 --- a/cmake/OWNERS +++ b/cmake/OWNERS @@ -1,4 +1,4 @@ set noparent @jtattermusch @nicolasnoble -@matt-kwong +@mehrdada diff --git a/config.m4 b/config.m4 index aa40a698a64..a46b076fe92 100644 --- a/config.m4 +++ b/config.m4 @@ -280,6 +280,8 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/security/credentials/plugin/plugin_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/security_connector/alts_security_connector.cc \ + src/core/lib/security/security_connector/load_system_roots_fallback.cc \ + src/core/lib/security/security_connector/load_system_roots_linux.cc \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/security_connector.cc \ src/core/lib/security/transport/client_auth_filter.cc \ diff --git a/config.w32 b/config.w32 index 5afa4466acd..3aea5fa7f20 100644 --- a/config.w32 +++ b/config.w32 @@ -255,6 +255,8 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\security\\credentials\\plugin\\plugin_credentials.cc " + "src\\core\\lib\\security\\credentials\\ssl\\ssl_credentials.cc " + "src\\core\\lib\\security\\security_connector\\alts_security_connector.cc " + + "src\\core\\lib\\security\\security_connector\\load_system_roots_fallback.cc " + + "src\\core\\lib\\security\\security_connector\\load_system_roots_linux.cc " + "src\\core\\lib\\security\\security_connector\\local_security_connector.cc " + "src\\core\\lib\\security\\security_connector\\security_connector.cc " + "src\\core\\lib\\security\\transport\\client_auth_filter.cc " + diff --git a/doc/csharp/server_reflection.md b/doc/csharp/server_reflection.md index 97216802694..7827cbb21f0 100644 --- a/doc/csharp/server_reflection.md +++ b/doc/csharp/server_reflection.md @@ -30,7 +30,8 @@ server.Start(); ``` After starting the server, you can verify that the server reflection -is working properly by using the `grpc_cli` command line tool: +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): ```sh $ grpc_cli ls localhost:50051 diff --git a/doc/python/server_reflection.md b/doc/python/server_reflection.md new file mode 100644 index 00000000000..5a68e3c3019 --- /dev/null +++ b/doc/python/server_reflection.md @@ -0,0 +1,61 @@ +# gRPC Python Server Reflection + +This document shows how to use gRPC Server Reflection in gRPC Python. +Please see [C++ Server Reflection Tutorial](../server_reflection_tutorial.md) +for general information and more examples how to use server reflection. + +## Enable server reflection in Python servers + +gRPC Python Server Reflection is an add-on library. +To use it, first install the [grpcio-reflection](https://pypi.org/project/grpcio-reflection/) +PyPI package into your project. + +Note that with Python you need to manually register the service +descriptors with the reflection service implementation when creating a server +(this isn't necessary with e.g. C++ or Java) +```python +# add the following import statement to use server reflection +from grpc_reflection.v1alpha import reflection +# ... +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + # the reflection service will be aware of "Greeter" and "ServerReflection" services. + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() +``` + +Please see +[greeter_server_with_reflection.py](https://github.com/grpc/grpc/blob/master/examples/python/helloworld/greeter_server_with_reflection.py) +in the examples directory for the full example, which extends the gRPC [Python +`Greeter` example](https://github.com/grpc/tree/master/examples/python/helloworld) on a +reflection-enabled server. + +After starting the server, you can verify that the server reflection +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): + + ```sh + $ grpc_cli ls localhost:50051 + ``` + + output: + ```sh + grpc.reflection.v1alpha.ServerReflection + helloworld.Greeter + ``` + + For more examples and instructions how to use the `grpc_cli` tool, + please refer to the [`grpc_cli` documentation](../command_line_tool.md) + and the [C++ Server Reflection Tutorial](../server_reflection_tutorial.md). + +## Additional Resources + +The [Server Reflection Protocol](../server-reflection.md) provides detailed +information about how the server reflection works and describes the server reflection +protocol in detail. diff --git a/doc/server-reflection.md b/doc/server-reflection.md index c9b476599f1..3716dc5a21d 100644 --- a/doc/server-reflection.md +++ b/doc/server-reflection.md @@ -191,6 +191,6 @@ each language: - [Go](https://github.com/grpc/grpc-go/blob/master/Documentation/server-reflection-tutorial.md#enable-server-reflection) - [C++](https://grpc.io/grpc/cpp/md_doc_server_reflection_tutorial.html) - [C#](https://github.com/grpc/grpc/blob/master/doc/csharp/server_reflection.md) -- Python: (tutorial not yet written) +- [Python](https://github.com/grpc/grpc/blob/master/doc/python/server_reflection.md) - Ruby: not yet implemented [#2567](https://github.com/grpc/grpc/issues/2567) - Node: not yet implemented [#2568](https://github.com/grpc/grpc/issues/2568) diff --git a/doc/ssl-performance.md b/doc/ssl-performance.md new file mode 100644 index 00000000000..176c8d8f247 --- /dev/null +++ b/doc/ssl-performance.md @@ -0,0 +1,36 @@ +# SSL in gRPC and performance + +The SSL requirement of gRPC isn't necessarily making it easy to integrate. The HTTP/2 protocol requires ALPN support, which is a fairly new handshake protocol only supported by recent implementations. + +As a result, we've tried hard to provide a smooth experience to our users when compiling and distributing gRPC, but this may come at performance costs due to this. More specifically, we will sometime build the SSL library by disabling assembly code +(by setting the `OPENSSL_NO_ASM` option), which can impact performance by an order of magnitude when processing encrypted streams. + +## gRPC C++: Building from Source + +Build system | Condition | Platform | Uses assembly optimizations +---|---|---|-- +Makefile | with OpenSSL 1.0.2 development files | all | :heavy_check_mark: +Makefile | all other cases | all | :x: +Bazel | | Linux | :heavy_check_mark: +Bazel | | MacOS | :heavy_check_mark: +Bazel | | Windows | :x: +CMake | boringssl from submodule (default) | all | :x: +CMake | pre-installed OpenSSL 1.0.2+ (`gRPC_SSL_PROVIDER=package`) | all | :heavy_check_mark: + +## Other Languages: Binary/Source Packages + +In addition, we are shipping packages for language implementations. These packages are source packages, but also have pre-built binaries being distributed. Building packages from source may give a different result in some cases. + +Language | From source | Platform | Uses assembly optimizations +---|---|---|--- +C# | n/a | all | :x: +Node.JS | n/a | Linux | :heavy_check_mark: +Node.JS | n/a | MacOS | :heavy_check_mark: +Node.JS | n/a | Windows | :x: +Electron | n/a | all | :heavy_check_mark: +ObjC | Yes | iOS | :x: +PHP | Yes | all | Same as the `Makefile` case from above +PHP | No | all | :x: +Python | n/a | all | :x: +Ruby | No | all | :x: + diff --git a/etc/roots.pem b/etc/roots.pem index 5dbd1ae6edb..c22dfe69f87 100644 --- a/etc/roots.pem +++ b/etc/roots.pem @@ -3734,169 +3734,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR -----END CERTIFICATE----- -# Issuer: CN=Certplus Root CA G1 O=Certplus -# Subject: CN=Certplus Root CA G1 O=Certplus -# Label: "Certplus Root CA G1" -# Serial: 1491911565779898356709731176965615564637713 -# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42 -# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66 -# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e ------BEGIN CERTIFICATE----- -MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa -MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy -dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB -ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a -iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt -6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP -0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f -6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE -EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN -1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc -h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT -mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV -4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO -WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud -DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd -Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq -hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh -66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7 -/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS -S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j -2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R -Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr -RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy -6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV -V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5 -g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl -++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo= ------END CERTIFICATE----- - -# Issuer: CN=Certplus Root CA G2 O=Certplus -# Subject: CN=Certplus Root CA G2 O=Certplus -# Label: "Certplus Root CA G2" -# Serial: 1492087096131536844209563509228951875861589 -# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31 -# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a -# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17 ------BEGIN CERTIFICATE----- -MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x -CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs -dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat -93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x -Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P -AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj -FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG -SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch -p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal -U5ORGpOucGpnutee5WEaXw== ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust -# Subject: CN=OpenTrust Root CA G1 O=OpenTrust -# Label: "OpenTrust Root CA G1" -# Serial: 1492036577811947013770400127034825178844775 -# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da -# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e -# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b -wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX -/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0 -77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP -uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx -p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx -Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2 -TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W -G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw -vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY -EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1 -2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw -DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E -PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf -gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS -FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0 -V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P -XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I -i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t -TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91 -09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky -Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ -AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj -1oxx ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust -# Subject: CN=OpenTrust Root CA G2 O=OpenTrust -# Label: "OpenTrust Root CA G2" -# Serial: 1492012448042702096986875987676935573415441 -# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb -# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b -# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2 ------BEGIN CERTIFICATE----- -MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA -MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w -ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw -MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU -T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK -AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh -/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e -CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6 -1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE -FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS -gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X -G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy -YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH -vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4 -t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/ -gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO -BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3 -5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w -DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz -Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0 -nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT -RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT -wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2 -t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa -TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2 -o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU -3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA -iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f -WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM -S1IK ------END CERTIFICATE----- - -# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust -# Subject: CN=OpenTrust Root CA G3 O=OpenTrust -# Label: "OpenTrust Root CA G3" -# Serial: 1492104908271485653071219941864171170455615 -# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24 -# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6 -# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92 ------BEGIN CERTIFICATE----- -MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx -CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U -cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow -QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl -blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm -3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d -oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G -A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5 -DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK -BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q -j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx -4nxp5V2a+EEfOzmTk51V6s2N8fvB ------END CERTIFICATE----- - # Issuer: CN=ISRG Root X1 O=Internet Security Research Group # Subject: CN=ISRG Root X1 O=Internet Security Research Group # Label: "ISRG Root X1" @@ -4440,3 +4277,43 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg== -----END CERTIFICATE----- + +# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6 +# Label: "GlobalSign Root CA - R6" +# Serial: 1417766617973444989252670301619537 +# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae +# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1 +# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69 +-----BEGIN CERTIFICATE----- +MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg +MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh +bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx +MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET +MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ +KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI +xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k +ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD +aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw +LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw +1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX +k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2 +SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h +bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n +WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY +rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce +MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD +AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu +bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN +nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt +Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61 +55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj +vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf +cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz +oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp +nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs +pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v +JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R +8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4 +5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA= +-----END CERTIFICATE----- diff --git a/examples/csharp/HelloworldXamarin/README.md b/examples/csharp/HelloworldXamarin/README.md index 4394dcc0c43..e47855de5e1 100644 --- a/examples/csharp/HelloworldXamarin/README.md +++ b/examples/csharp/HelloworldXamarin/README.md @@ -8,27 +8,29 @@ The example depends on experimental Grpc.Core nuget package that hasn't been officially released and is only available via the [daily builds](https://packages.grpc.io/) source. +HINT: To download the package, please manually download the latest `.nupkg` packages from "Daily Builds" in [packages.grpc.io](https://packages.grpc.io/) into a local directory. Then add a nuget source that points to that directory (That can be [done in Visual Studio](https://docs.microsoft.com/en-us/nuget/tools/package-manager-ui#package-sources) or Visual Studio for Mac via "Configure nuget sources"). After that, nuget will also explore that directory when looking for packages. + BACKGROUND ------------- -The example project supports Xamarin.Android and Xamarin.iOS +The example project supports `Xamarin.Android` and `Xamarin.iOS`. For this sample, we've already generated the server and client stubs from [helloworld.proto][]. PREREQUISITES ------------- -- The latest version Xamarin Studio or Visual Studio 2017 with Xamarin support installed. +- The latest version Visual Studio 2017 or Visual Studio for Mac with Xamarin support installed. BUILD ------- -- Open the `HelloworldXamarin.sln` in Visual Studio (or Xamarin Studio) +- Open the `HelloworldXamarin.sln` in Visual Studio (or Visual Studio for Mac) - Build the solution (Build -> Build All) Try it! ------- -You can deploy the example apps directly through Xamarin Studio IDE. +You can deploy the example apps directly through Visual Studio IDE. Deployments can target both Android and iOS (both support physical device deployment as well as simulator). diff --git a/examples/python/helloworld/greeter_server_with_reflection.py b/examples/python/helloworld/greeter_server_with_reflection.py new file mode 100644 index 00000000000..5ba8782dfca --- /dev/null +++ b/examples/python/helloworld/greeter_server_with_reflection.py @@ -0,0 +1,52 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The reflection-enabled version of gRPC helloworld.Greeter server.""" + +from concurrent import futures +import time + +import grpc +from grpc_reflection.v1alpha import reflection + +import helloworld_pb2 +import helloworld_pb2_grpc + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, context): + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == '__main__': + serve() diff --git a/examples/python/multiplex/helloworld_pb2.py b/examples/python/multiplex/helloworld_pb2.py index 6665b1f6878..e18ab9acc7a 100644 --- a/examples/python/multiplex/helloworld_pb2.py +++ b/examples/python/multiplex/helloworld_pb2.py @@ -21,7 +21,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_pb=_b('\n\x10helloworld.proto\x12\nhelloworld\"\x1c\n\x0cHelloRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"\x1d\n\nHelloReply\x12\x0f\n\x07message\x18\x01 \x01(\t2I\n\x07Greeter\x12>\n\x08SayHello\x12\x18.helloworld.HelloRequest\x1a\x16.helloworld.HelloReply\"\x00\x42\x36\n\x1bio.grpc.examples.helloworldB\x0fHelloWorldProtoP\x01\xa2\x02\x03HLWb\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -89,6 +88,7 @@ _HELLOREPLY = _descriptor.Descriptor( DESCRIPTOR.message_types_by_name['HelloRequest'] = _HELLOREQUEST DESCRIPTOR.message_types_by_name['HelloReply'] = _HELLOREPLY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) HelloRequest = _reflection.GeneratedProtocolMessageType('HelloRequest', (_message.Message,), dict( DESCRIPTOR = _HELLOREQUEST, @@ -107,123 +107,28 @@ _sym_db.RegisterMessage(HelloReply) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.helloworldB\017HelloWorldProtoP\001\242\002\003HLW')) -try: - # THESE ELEMENTS WILL BE DEPRECATED. - # Please use the generated *_pb2_grpc.py files instead. - import grpc - from grpc.framework.common import cardinality - from grpc.framework.interfaces.face import utilities as face_utilities - from grpc.beta import implementations as beta_implementations - from grpc.beta import interfaces as beta_interfaces - - - class GreeterStub(object): - """The greeting service definition. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.SayHello = channel.unary_unary( - '/helloworld.Greeter/SayHello', - request_serializer=HelloRequest.SerializeToString, - response_deserializer=HelloReply.FromString, - ) - - - class GreeterServicer(object): - """The greeting service definition. - """ - - def SayHello(self, request, context): - """Sends a greeting - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - - def add_GreeterServicer_to_server(servicer, server): - rpc_method_handlers = { - 'SayHello': grpc.unary_unary_rpc_method_handler( - servicer.SayHello, - request_deserializer=HelloRequest.FromString, - response_serializer=HelloReply.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'helloworld.Greeter', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - class BetaGreeterServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """The greeting service definition. - """ - def SayHello(self, request, context): - """Sends a greeting - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - - - class BetaGreeterStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """The greeting service definition. - """ - def SayHello(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """Sends a greeting - """ - raise NotImplementedError() - SayHello.future = None - - - def beta_create_Greeter_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('helloworld.Greeter', 'SayHello'): HelloRequest.FromString, - } - response_serializers = { - ('helloworld.Greeter', 'SayHello'): HelloReply.SerializeToString, - } - method_implementations = { - ('helloworld.Greeter', 'SayHello'): face_utilities.unary_unary_inline(servicer.SayHello), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - - def beta_create_Greeter_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('helloworld.Greeter', 'SayHello'): HelloRequest.SerializeToString, - } - response_deserializers = { - ('helloworld.Greeter', 'SayHello'): HelloReply.FromString, - } - cardinalities = { - 'SayHello': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'helloworld.Greeter', cardinalities, options=stub_options) -except ImportError: - pass + +_GREETER = _descriptor.ServiceDescriptor( + name='Greeter', + full_name='helloworld.Greeter', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=93, + serialized_end=166, + methods=[ + _descriptor.MethodDescriptor( + name='SayHello', + full_name='helloworld.Greeter.SayHello', + index=0, + containing_service=None, + input_type=_HELLOREQUEST, + output_type=_HELLOREPLY, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_GREETER) + +DESCRIPTOR.services_by_name['Greeter'] = _GREETER + # @@protoc_insertion_point(module_scope) diff --git a/examples/python/multiplex/helloworld_pb2_grpc.py b/examples/python/multiplex/helloworld_pb2_grpc.py index 682dc36cd89..18e07d16797 100644 --- a/examples/python/multiplex/helloworld_pb2_grpc.py +++ b/examples/python/multiplex/helloworld_pb2_grpc.py @@ -1,6 +1,5 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities import helloworld_pb2 as helloworld__pb2 diff --git a/examples/python/multiplex/route_guide_pb2.py b/examples/python/multiplex/route_guide_pb2.py index e6775eb8140..fbbc194a580 100644 --- a/examples/python/multiplex/route_guide_pb2.py +++ b/examples/python/multiplex/route_guide_pb2.py @@ -21,7 +21,6 @@ DESCRIPTOR = _descriptor.FileDescriptor( syntax='proto3', serialized_pb=_b('\n\x11route_guide.proto\x12\nrouteguide\",\n\x05Point\x12\x10\n\x08latitude\x18\x01 \x01(\x05\x12\x11\n\tlongitude\x18\x02 \x01(\x05\"I\n\tRectangle\x12\x1d\n\x02lo\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x1d\n\x02hi\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"<\n\x07\x46\x65\x61ture\x12\x0c\n\x04name\x18\x01 \x01(\t\x12#\n\x08location\x18\x02 \x01(\x0b\x32\x11.routeguide.Point\"A\n\tRouteNote\x12#\n\x08location\x18\x01 \x01(\x0b\x32\x11.routeguide.Point\x12\x0f\n\x07message\x18\x02 \x01(\t\"b\n\x0cRouteSummary\x12\x13\n\x0bpoint_count\x18\x01 \x01(\x05\x12\x15\n\rfeature_count\x18\x02 \x01(\x05\x12\x10\n\x08\x64istance\x18\x03 \x01(\x05\x12\x14\n\x0c\x65lapsed_time\x18\x04 \x01(\x05\x32\x85\x02\n\nRouteGuide\x12\x36\n\nGetFeature\x12\x11.routeguide.Point\x1a\x13.routeguide.Feature\"\x00\x12>\n\x0cListFeatures\x12\x15.routeguide.Rectangle\x1a\x13.routeguide.Feature\"\x00\x30\x01\x12>\n\x0bRecordRoute\x12\x11.routeguide.Point\x1a\x18.routeguide.RouteSummary\"\x00(\x01\x12?\n\tRouteChat\x12\x15.routeguide.RouteNote\x1a\x15.routeguide.RouteNote\"\x00(\x01\x30\x01\x42\x36\n\x1bio.grpc.examples.routeguideB\x0fRouteGuideProtoP\x01\xa2\x02\x03RTGb\x06proto3') ) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -238,6 +237,7 @@ DESCRIPTOR.message_types_by_name['Rectangle'] = _RECTANGLE DESCRIPTOR.message_types_by_name['Feature'] = _FEATURE DESCRIPTOR.message_types_by_name['RouteNote'] = _ROUTENOTE DESCRIPTOR.message_types_by_name['RouteSummary'] = _ROUTESUMMARY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) Point = _reflection.GeneratedProtocolMessageType('Point', (_message.Message,), dict( DESCRIPTOR = _POINT, @@ -277,265 +277,55 @@ _sym_db.RegisterMessage(RouteSummary) DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\033io.grpc.examples.routeguideB\017RouteGuideProtoP\001\242\002\003RTG')) -try: - # THESE ELEMENTS WILL BE DEPRECATED. - # Please use the generated *_pb2_grpc.py files instead. - import grpc - from grpc.framework.common import cardinality - from grpc.framework.interfaces.face import utilities as face_utilities - from grpc.beta import implementations as beta_implementations - from grpc.beta import interfaces as beta_interfaces - - - class RouteGuideStub(object): - """Interface exported by the server. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.GetFeature = channel.unary_unary( - '/routeguide.RouteGuide/GetFeature', - request_serializer=Point.SerializeToString, - response_deserializer=Feature.FromString, - ) - self.ListFeatures = channel.unary_stream( - '/routeguide.RouteGuide/ListFeatures', - request_serializer=Rectangle.SerializeToString, - response_deserializer=Feature.FromString, - ) - self.RecordRoute = channel.stream_unary( - '/routeguide.RouteGuide/RecordRoute', - request_serializer=Point.SerializeToString, - response_deserializer=RouteSummary.FromString, - ) - self.RouteChat = channel.stream_stream( - '/routeguide.RouteGuide/RouteChat', - request_serializer=RouteNote.SerializeToString, - response_deserializer=RouteNote.FromString, - ) - - - class RouteGuideServicer(object): - """Interface exported by the server. - """ - - def GetFeature(self, request, context): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ListFeatures(self, request, context): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def RecordRoute(self, request_iterator, context): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def RouteChat(self, request_iterator, context): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - - def add_RouteGuideServicer_to_server(servicer, server): - rpc_method_handlers = { - 'GetFeature': grpc.unary_unary_rpc_method_handler( - servicer.GetFeature, - request_deserializer=Point.FromString, - response_serializer=Feature.SerializeToString, - ), - 'ListFeatures': grpc.unary_stream_rpc_method_handler( - servicer.ListFeatures, - request_deserializer=Rectangle.FromString, - response_serializer=Feature.SerializeToString, - ), - 'RecordRoute': grpc.stream_unary_rpc_method_handler( - servicer.RecordRoute, - request_deserializer=Point.FromString, - response_serializer=RouteSummary.SerializeToString, - ), - 'RouteChat': grpc.stream_stream_rpc_method_handler( - servicer.RouteChat, - request_deserializer=RouteNote.FromString, - response_serializer=RouteNote.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'routeguide.RouteGuide', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - class BetaRouteGuideServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """Interface exported by the server. - """ - def GetFeature(self, request, context): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def ListFeatures(self, request, context): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def RecordRoute(self, request_iterator, context): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def RouteChat(self, request_iterator, context): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - - - class BetaRouteGuideStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """Interface exported by the server. - """ - def GetFeature(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """A simple RPC. - - Obtains the feature at a given position. - - A feature with an empty name is returned if there's no feature at the given - position. - """ - raise NotImplementedError() - GetFeature.future = None - def ListFeatures(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """A server-to-client streaming RPC. - - Obtains the Features available within the given Rectangle. Results are - streamed rather than returned at once (e.g. in a response message with a - repeated field), as the rectangle may cover a large area and contain a - huge number of features. - """ - raise NotImplementedError() - def RecordRoute(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): - """A client-to-server streaming RPC. - - Accepts a stream of Points on a route being traversed, returning a - RouteSummary when traversal is completed. - """ - raise NotImplementedError() - RecordRoute.future = None - def RouteChat(self, request_iterator, timeout, metadata=None, with_call=False, protocol_options=None): - """A Bidirectional streaming RPC. - - Accepts a stream of RouteNotes sent while a route is being traversed, - while receiving other RouteNotes (e.g. from other users). - """ - raise NotImplementedError() - - - def beta_create_RouteGuide_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('routeguide.RouteGuide', 'GetFeature'): Point.FromString, - ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.FromString, - ('routeguide.RouteGuide', 'RecordRoute'): Point.FromString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString, - } - response_serializers = { - ('routeguide.RouteGuide', 'GetFeature'): Feature.SerializeToString, - ('routeguide.RouteGuide', 'ListFeatures'): Feature.SerializeToString, - ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.SerializeToString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString, - } - method_implementations = { - ('routeguide.RouteGuide', 'GetFeature'): face_utilities.unary_unary_inline(servicer.GetFeature), - ('routeguide.RouteGuide', 'ListFeatures'): face_utilities.unary_stream_inline(servicer.ListFeatures), - ('routeguide.RouteGuide', 'RecordRoute'): face_utilities.stream_unary_inline(servicer.RecordRoute), - ('routeguide.RouteGuide', 'RouteChat'): face_utilities.stream_stream_inline(servicer.RouteChat), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - - def beta_create_RouteGuide_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. +_ROUTEGUIDE = _descriptor.ServiceDescriptor( + name='RouteGuide', + full_name='routeguide.RouteGuide', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=384, + serialized_end=645, + methods=[ + _descriptor.MethodDescriptor( + name='GetFeature', + full_name='routeguide.RouteGuide.GetFeature', + index=0, + containing_service=None, + input_type=_POINT, + output_type=_FEATURE, + options=None, + ), + _descriptor.MethodDescriptor( + name='ListFeatures', + full_name='routeguide.RouteGuide.ListFeatures', + index=1, + containing_service=None, + input_type=_RECTANGLE, + output_type=_FEATURE, + options=None, + ), + _descriptor.MethodDescriptor( + name='RecordRoute', + full_name='routeguide.RouteGuide.RecordRoute', + index=2, + containing_service=None, + input_type=_POINT, + output_type=_ROUTESUMMARY, + options=None, + ), + _descriptor.MethodDescriptor( + name='RouteChat', + full_name='routeguide.RouteGuide.RouteChat', + index=3, + containing_service=None, + input_type=_ROUTENOTE, + output_type=_ROUTENOTE, + options=None, + ), +]) +_sym_db.RegisterServiceDescriptor(_ROUTEGUIDE) + +DESCRIPTOR.services_by_name['RouteGuide'] = _ROUTEGUIDE - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('routeguide.RouteGuide', 'GetFeature'): Point.SerializeToString, - ('routeguide.RouteGuide', 'ListFeatures'): Rectangle.SerializeToString, - ('routeguide.RouteGuide', 'RecordRoute'): Point.SerializeToString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.SerializeToString, - } - response_deserializers = { - ('routeguide.RouteGuide', 'GetFeature'): Feature.FromString, - ('routeguide.RouteGuide', 'ListFeatures'): Feature.FromString, - ('routeguide.RouteGuide', 'RecordRoute'): RouteSummary.FromString, - ('routeguide.RouteGuide', 'RouteChat'): RouteNote.FromString, - } - cardinalities = { - 'GetFeature': cardinality.Cardinality.UNARY_UNARY, - 'ListFeatures': cardinality.Cardinality.UNARY_STREAM, - 'RecordRoute': cardinality.Cardinality.STREAM_UNARY, - 'RouteChat': cardinality.Cardinality.STREAM_STREAM, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'routeguide.RouteGuide', cardinalities, options=stub_options) -except ImportError: - pass # @@protoc_insertion_point(module_scope) diff --git a/examples/python/multiplex/route_guide_pb2_grpc.py b/examples/python/multiplex/route_guide_pb2_grpc.py index 27b24c747db..05c1b793128 100644 --- a/examples/python/multiplex/route_guide_pb2_grpc.py +++ b/examples/python/multiplex/route_guide_pb2_grpc.py @@ -1,6 +1,5 @@ +# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities import route_guide_pb2 as route__guide__pb2 diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 1d9237bf627..1d3cedb16be 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -278,6 +278,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 5c3649afbde..1998bc8b4c5 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -289,6 +289,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', @@ -705,6 +707,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', @@ -882,6 +886,8 @@ Pod::Spec.new do |s| 'src/core/lib/security/credentials/plugin/plugin_credentials.h', 'src/core/lib/security/credentials/ssl/ssl_credentials.h', 'src/core/lib/security/security_connector/alts_security_connector.h', + 'src/core/lib/security/security_connector/load_system_roots.h', + 'src/core/lib/security/security_connector/load_system_roots_linux.h', 'src/core/lib/security/security_connector/local_security_connector.h', 'src/core/lib/security/security_connector/security_connector.h', 'src/core/lib/security/transport/auth_filters.h', diff --git a/grpc.def b/grpc.def index 5d699e890a8..356137f80aa 100644 --- a/grpc.def +++ b/grpc.def @@ -42,6 +42,7 @@ EXPORTS grpc_census_call_get_context grpc_channel_get_target grpc_channel_get_info + grpc_channel_reset_connect_backoff grpc_insecure_channel_create grpc_lame_client_channel_create grpc_channel_destroy diff --git a/grpc.gemspec b/grpc.gemspec index c250316b995..55d53cb71db 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -222,6 +222,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.h ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.h ) s.files += %w( src/core/lib/security/security_connector/alts_security_connector.h ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots.h ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_linux.h ) s.files += %w( src/core/lib/security/security_connector/local_security_connector.h ) s.files += %w( src/core/lib/security/security_connector/security_connector.h ) s.files += %w( src/core/lib/security/transport/auth_filters.h ) @@ -642,6 +644,8 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.cc ) s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.cc ) s.files += %w( src/core/lib/security/security_connector/alts_security_connector.cc ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_fallback.cc ) + s.files += %w( src/core/lib/security/security_connector/load_system_roots_linux.cc ) s.files += %w( src/core/lib/security/security_connector/local_security_connector.cc ) s.files += %w( src/core/lib/security/security_connector/security_connector.cc ) s.files += %w( src/core/lib/security/transport/client_auth_filter.cc ) diff --git a/grpc.gyp b/grpc.gyp index 25082fe540a..ba4e8185c61 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -472,6 +472,8 @@ 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index 7c4307e83b7..412778f7f73 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -274,6 +274,11 @@ GRPCAPI char* grpc_channel_get_target(grpc_channel* channel); GRPCAPI void grpc_channel_get_info(grpc_channel* channel, const grpc_channel_info* channel_info); +/** EXPERIMENTAL. Resets the channel's connect backoff. + TODO(roth): When we see whether this proves useful, either promote + to non-experimental or remove it. */ +GRPCAPI void grpc_channel_reset_connect_backoff(grpc_channel* channel); + /** Create a client channel to 'target'. Additional channel level configuration MAY be provided by grpc_channel_args, though the expectation is that most clients will want to simply pass NULL. The user data in 'args' need only diff --git a/include/grpcpp/channel.h b/include/grpcpp/channel.h index 4b45d5382cd..fed02bf7bc7 100644 --- a/include/grpcpp/channel.h +++ b/include/grpcpp/channel.h @@ -30,6 +30,14 @@ struct grpc_channel; namespace grpc { + +namespace experimental { +/// Resets the channel's connection backoff. +/// TODO(roth): Once we see whether this proves useful, either create a gRFC +/// and change this to be a method of the Channel class, or remove it. +void ChannelResetConnectionBackoff(Channel* channel); +} // namespace experimental + /// Channels represent a connection to an endpoint. Created by \a CreateChannel. class Channel final : public ChannelInterface, public internal::CallHook, @@ -52,6 +60,7 @@ class Channel final : public ChannelInterface, private: template friend class internal::BlockingUnaryCallImpl; + friend void experimental::ChannelResetConnectionBackoff(Channel* channel); friend std::shared_ptr CreateChannelInternal( const grpc::string& host, grpc_channel* c_channel); Channel(const grpc::string& host, grpc_channel* c_channel); diff --git a/include/grpcpp/impl/codegen/async_generic_service.h b/include/grpcpp/impl/codegen/async_generic_service.h index 957bb776f1d..2a0e1b40881 100644 --- a/include/grpcpp/impl/codegen/async_generic_service.h +++ b/include/grpcpp/impl/codegen/async_generic_service.h @@ -52,12 +52,12 @@ class GenericServerContext final : public ServerContext { // ServerBuilder builder; // auto cq = builder.AddCompletionQueue(); // AsyncGenericService generic_service; -// builder.RegisterAsyncGeneicService(&generic_service); +// builder.RegisterAsyncGenericService(&generic_service); // auto server = builder.BuildAndStart(); // // // request a new call // GenericServerContext context; -// GenericAsyncReaderWriter stream; +// GenericServerAsyncReaderWriter stream; // generic_service.RequestCall(&context, &stream, cq.get(), cq.get(), tag); // // When tag is retrieved from cq->Next(), context.method() can be used to look diff --git a/package.xml b/package.xml index acdc6ffdb38..76bdd5ac8f6 100644 --- a/package.xml +++ b/package.xml @@ -227,6 +227,8 @@ + + @@ -647,6 +649,8 @@ + + diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index 4263e791c4a..82c844a8003 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -622,6 +622,17 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { } GRPC_ERROR_UNREF(op->disconnect_with_error); } + + if (op->reset_connect_backoff) { + if (chand->resolver != nullptr) { + chand->resolver->ResetBackoffLocked(); + chand->resolver->RequestReresolutionLocked(); + } + if (chand->lb_policy != nullptr) { + chand->lb_policy->ResetBackoffLocked(); + } + } + GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "start_transport_op"); GRPC_CLOSURE_SCHED(op->on_consumed, GRPC_ERROR_NONE); diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.cc b/src/core/ext/filters/client_channel/client_channel_plugin.cc index e0784b7e5c1..71da6486604 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.cc +++ b/src/core/ext/filters/client_channel/client_channel_plugin.cc @@ -56,7 +56,7 @@ void grpc_client_channel_init(void) { grpc_register_http_proxy_mapper(); grpc_subchannel_index_init(); grpc_channel_init_register_stage( - GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, append_filter, + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, append_filter, (void*)&grpc_client_channel_filter); grpc_http_connect_register_handshaker_factory(); } diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 31c08246ae4..3c0a9c11180 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -144,7 +144,10 @@ class LoadBalancingPolicy /// consider whether this method is still needed. virtual void ExitIdleLocked() GRPC_ABSTRACT; - /// populates child_subchannels and child_channels with the uuids of this + /// Resets connection backoff. + virtual void ResetBackoffLocked() GRPC_ABSTRACT; + + /// Populates child_subchannels and child_channels with the uuids of this /// LB policy's referenced children. This is not invoked from the /// client_channel's combiner. The implementation is responsible for /// providing its own synchronization. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 9e83abfebb9..b1d97a70165 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -134,6 +134,7 @@ class GrpcLb : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) override; @@ -1214,6 +1215,15 @@ void GrpcLb::ExitIdleLocked() { } } +void GrpcLb::ResetBackoffLocked() { + if (lb_channel_ != nullptr) { + grpc_channel_reset_connect_backoff(lb_channel_); + } + if (rr_policy_ != nullptr) { + rr_policy_->ResetBackoffLocked(); + } +} + bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) { PendingPick* pp = PendingPickCreate(pick); bool pick_done = false; @@ -1880,7 +1890,7 @@ void grpc_lb_policy_grpclb_init() { grpc_core::UniquePtr( grpc_core::New())); grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_client_load_reporting_filter, (void*)&grpc_client_load_reporting_filter); } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 46acbf628b0..2b6a9ba8c53 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -57,6 +57,7 @@ class PickFirst : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -259,6 +260,13 @@ void PickFirst::ExitIdleLocked() { } } +void PickFirst::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool PickFirst::PickLocked(PickState* pick, grpc_error** error) { // If we have a selected subchannel already, return synchronously. if (selected_ != nullptr) { diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 9c3a15c67b0..fea84331d80 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -68,6 +68,7 @@ class RoundRobin : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -333,6 +334,13 @@ void RoundRobin::ExitIdleLocked() { } } +void RoundRobin::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool RoundRobin::DoPickLocked(PickState* pick) { const size_t next_ready_index = subchannel_list_->GetNextReadySubchannelIndexLocked(); diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 199b9a3c13e..d4e51c584a4 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -107,6 +107,11 @@ class SubchannelData { // being unreffed. virtual void UnrefSubchannelLocked(const char* reason); + // Resets the connection backoff. + // TODO(roth): This method should go away when we move the backoff + // code out of the subchannel and into the LB policies. + void ResetBackoffLocked(); + // Starts watching the connectivity state of the subchannel. // ProcessConnectivityChangeLocked() will be called when the // connectivity state changes. @@ -206,6 +211,11 @@ class SubchannelList LoadBalancingPolicy* policy() const { return policy_; } TraceFlag* tracer() const { return tracer_; } + // Resets connection backoff of all subchannels. + // TODO(roth): We will probably need to rethink this as part of moving + // the backoff code out of subchannels and into LB policies. + void ResetBackoffLocked(); + // Note: Caller must ensure that this is invoked inside of the combiner. void Orphan() override { ShutdownLocked(); @@ -298,6 +308,14 @@ void SubchannelData:: } } +template +void SubchannelData::ResetBackoffLocked() { + if (subchannel_ != nullptr) { + grpc_subchannel_reset_backoff(subchannel_); + } +} + template void SubchannelData::StartConnectivityWatchLocked() { @@ -544,6 +562,15 @@ void SubchannelList::ShutdownLocked() { } } +template +void SubchannelList::ResetBackoffLocked() { + for (size_t i = 0; i < subchannels_.size(); i++) { + SubchannelDataType* sd = &subchannels_[i]; + sd->ResetBackoffLocked(); + } +} + } // namespace grpc_core #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_SUBCHANNEL_LIST_H */ diff --git a/src/core/ext/filters/client_channel/resolver.h b/src/core/ext/filters/client_channel/resolver.h index c7e37e4468e..48f2e890952 100644 --- a/src/core/ext/filters/client_channel/resolver.h +++ b/src/core/ext/filters/client_channel/resolver.h @@ -94,6 +94,14 @@ class Resolver : public InternallyRefCountedWithTracing { /// throw away unselected subchannels. virtual void RequestReresolutionLocked() GRPC_ABSTRACT; + /// Resets the re-resolution backoff, if any. + /// This needs to be implemented only by pull-based implementations; + /// for push-based implementations, it will be a no-op. + /// TODO(roth): Pull the backoff code out of resolver and into + /// client_channel, so that it can be shared across resolver + /// implementations. At that point, this method can go away. + virtual void ResetBackoffLocked() {} + void Orphan() override { // Invoke ShutdownAndUnrefLocked() inside of the combiner. GRPC_CLOSURE_SCHED( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index 7050e82121c..f2bb5f3c712 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -66,6 +66,8 @@ class AresDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -187,6 +189,13 @@ void AresDnsResolver::RequestReresolutionLocked() { } } +void AresDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void AresDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc index 0068d0d5f4b..fdbd07ebf51 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc @@ -74,6 +74,8 @@ struct grpc_ares_ev_driver { bool shutting_down; /** request object that's using this ev driver */ grpc_ares_request* request; + /** Owned by the ev_driver. Creates new GrpcPolledFd's */ + grpc_core::UniquePtr polled_fd_factory; }; static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver); @@ -93,7 +95,7 @@ static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) { GRPC_COMBINER_UNREF(ev_driver->combiner, "free ares event driver"); ares_destroy(ev_driver->channel); grpc_ares_complete_request_locked(ev_driver->request); - gpr_free(ev_driver); + grpc_core::Delete(ev_driver); } } @@ -118,13 +120,11 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, grpc_pollset_set* pollset_set, grpc_combiner* combiner, grpc_ares_request* request) { - *ev_driver = static_cast( - gpr_malloc(sizeof(grpc_ares_ev_driver))); + *ev_driver = grpc_core::New(); ares_options opts; memset(&opts, 0, sizeof(opts)); opts.flags |= ARES_FLAG_STAYOPEN; int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS); - grpc_core::ConfigureAresChannelLocked(&(*ev_driver)->channel); gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create_locked"); if (status != ARES_SUCCESS) { char* err_msg; @@ -142,6 +142,10 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, (*ev_driver)->working = false; (*ev_driver)->shutting_down = false; (*ev_driver)->request = request; + (*ev_driver)->polled_fd_factory = + grpc_core::NewGrpcPolledFdFactory((*ev_driver)->combiner); + (*ev_driver) + ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel); return GRPC_ERROR_NONE; } @@ -245,8 +249,9 @@ static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) { // Create a new fd_node if sock[i] is not in the fd_node list. if (fdn == nullptr) { fdn = static_cast(gpr_malloc(sizeof(fd_node))); - fdn->grpc_polled_fd = grpc_core::NewGrpcPolledFdLocked( - socks[i], ev_driver->pollset_set); + fdn->grpc_polled_fd = + ev_driver->polled_fd_factory->NewGrpcPolledFdLocked( + socks[i], ev_driver->pollset_set, ev_driver->combiner); gpr_log(GPR_DEBUG, "new fd: %s", fdn->grpc_polled_fd->GetName()); fdn->ev_driver = ev_driver; fdn->readable_registered = false; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h index 2c9db710116..671c537fe72 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h @@ -81,10 +81,24 @@ class GrpcPolledFd { GRPC_ABSTRACT_BASE_CLASS }; -/* Creates a new wrapped fd for the current platform */ -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set); -void ConfigureAresChannelLocked(ares_channel* channel); +/* A GrpcPolledFdFactory is 1-to-1 with and owned by the + * ares event driver. It knows how to create GrpcPolledFd's + * for the current platform, and the ares driver uses it for all of + * its fd's. */ +class GrpcPolledFdFactory { + public: + virtual ~GrpcPolledFdFactory() {} + /* Creates a new wrapped fd for the current platform */ + virtual GrpcPolledFd* NewGrpcPolledFdLocked( + ares_socket_t as, grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) GRPC_ABSTRACT; + /* Optionally configures the ares channel after creation */ + virtual void ConfigureAresChannelLocked(ares_channel channel) GRPC_ABSTRACT; + + GRPC_ABSTRACT_BASE_CLASS +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner); } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc index fffe9eda8e5..aa58e1aaf50 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc @@ -86,12 +86,20 @@ class GrpcPolledFdPosix : public GrpcPolledFd { grpc_pollset_set* driver_pollset_set_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return grpc_core::New(as, driver_pollset_set); -} +class GrpcPolledFdFactoryPosix : public GrpcPolledFdFactory { + public: + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + return New(as, driver_pollset_set); + } -void ConfigureAresChannelLocked(ares_channel* channel) {} + void ConfigureAresChannelLocked(ares_channel channel) override {} +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr(New()); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc index 5d65ae3ab37..02121aa0ab4 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc @@ -21,38 +21,516 @@ #if GRPC_ARES == 1 && defined(GPR_WINDOWS) #include + +#include +#include +#include +#include +#include #include +#include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" + +/* TODO(apolcyn): remove this hack after fixing upstream. + * Our grpc/c-ares code on Windows uses the ares_set_socket_functions API, + * which uses "struct iovec" type, which on Windows is defined inside of + * a c-ares header that is not public. + * See https://github.com/c-ares/c-ares/issues/206. */ +struct iovec { + void* iov_base; + size_t iov_len; +}; namespace grpc_core { -/* TODO: fill in the body of GrpcPolledFdWindows to enable c-ares on Windows. - This dummy implementation only allows grpc to compile on windows with - GRPC_ARES=1. */ +/* c-ares creates its own sockets and is meant to read them when readable and + * write them when writeable. To fit this socket usage model into the grpc + * windows poller (which gives notifications when attempted reads and writes are + * actually fulfilled rather than possible), this GrpcPolledFdWindows class + * takes advantage of the ares_set_socket_functions API and acts as a virtual + * socket. It holds its own read and write buffers which are written to and read + * from c-ares and are used with the grpc windows poller, and it, e.g., + * manufactures virtual socket error codes when it e.g. needs to tell the c-ares + * library to wait for an async read. */ class GrpcPolledFdWindows : public GrpcPolledFd { public: - GrpcPolledFdWindows() { abort(); } - ~GrpcPolledFdWindows() { abort(); } + enum WriteState { + WRITE_IDLE, + WRITE_REQUESTED, + WRITE_PENDING, + WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY, + }; + + GrpcPolledFdWindows(ares_socket_t as, grpc_combiner* combiner) + : read_buf_(grpc_empty_slice()), + write_buf_(grpc_empty_slice()), + write_state_(WRITE_IDLE), + gotten_into_driver_list_(false) { + gpr_asprintf(&name_, "c-ares socket: %" PRIdPTR, as); + winsocket_ = grpc_winsocket_create(as, name_); + combiner_ = GRPC_COMBINER_REF(combiner, name_); + GRPC_CLOSURE_INIT(&outer_read_closure_, + &GrpcPolledFdWindows::OnIocpReadable, this, + grpc_combiner_scheduler(combiner_)); + GRPC_CLOSURE_INIT(&outer_write_closure_, + &GrpcPolledFdWindows::OnIocpWriteable, this, + grpc_combiner_scheduler(combiner_)); + } + + ~GrpcPolledFdWindows() { + GRPC_COMBINER_UNREF(combiner_, name_); + grpc_slice_unref_internal(read_buf_); + grpc_slice_unref_internal(write_buf_); + GPR_ASSERT(read_closure_ == nullptr); + GPR_ASSERT(write_closure_ == nullptr); + grpc_winsocket_destroy(winsocket_); + gpr_free(name_); + } + + void ScheduleAndNullReadClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(read_closure_, error); + read_closure_ = nullptr; + } + + void ScheduleAndNullWriteClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(write_closure_, error); + write_closure_ = nullptr; + } + void RegisterForOnReadableLocked(grpc_closure* read_closure) override { - abort(); + GPR_ASSERT(read_closure_ == nullptr); + read_closure_ = read_closure; + GPR_ASSERT(GRPC_SLICE_LENGTH(read_buf_) == 0); + grpc_slice_unref_internal(read_buf_); + read_buf_ = GRPC_SLICE_MALLOC(4192); + WSABUF buffer; + buffer.buf = (char*)GRPC_SLICE_START_PTR(read_buf_); + buffer.len = GRPC_SLICE_LENGTH(read_buf_); + memset(&winsocket_->read_info.overlapped, 0, sizeof(OVERLAPPED)); + recv_from_source_addr_len_ = sizeof(recv_from_source_addr_); + DWORD flags = 0; + if (WSARecvFrom(grpc_winsocket_wrapped_socket(winsocket_), &buffer, 1, + nullptr, &flags, (sockaddr*)recv_from_source_addr_, + &recv_from_source_addr_len_, + &winsocket_->read_info.overlapped, nullptr)) { + char* msg = gpr_format_message(WSAGetLastError()); + grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + GRPC_CARES_TRACE_LOG( + "RegisterForOnReadableLocked: WSARecvFrom error:|%s|. fd:|%s|", msg, + GetName()); + gpr_free(msg); + if (WSAGetLastError() != WSA_IO_PENDING) { + ScheduleAndNullReadClosure(error); + return; + } + } + grpc_socket_notify_on_read(winsocket_, &outer_read_closure_); } + void RegisterForOnWriteableLocked(grpc_closure* write_closure) override { + GRPC_CARES_TRACE_LOG( + "RegisterForOnWriteableLocked. fd:|%s|. Current write state: %d", + GetName(), write_state_); + GPR_ASSERT(write_closure_ == nullptr); + write_closure_ = write_closure; + switch (write_state_) { + case WRITE_IDLE: + ScheduleAndNullWriteClosure(GRPC_ERROR_NONE); + break; + case WRITE_REQUESTED: + write_state_ = WRITE_PENDING; + SendWriteBuf(nullptr, &winsocket_->write_info.overlapped); + grpc_socket_notify_on_write(winsocket_, &outer_write_closure_); + break; + case WRITE_PENDING: + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + abort(); + } + } + + bool IsFdStillReadableLocked() override { + return GRPC_SLICE_LENGTH(read_buf_) > 0; + } + + void ShutdownLocked(grpc_error* error) override { + grpc_winsocket_shutdown(winsocket_); + } + + ares_socket_t GetWrappedAresSocketLocked() override { + return grpc_winsocket_wrapped_socket(winsocket_); + } + + const char* GetName() override { return name_; } + + ares_ssize_t RecvFrom(void* data, ares_socket_t data_len, int flags, + struct sockaddr* from, ares_socklen_t* from_len) { + GRPC_CARES_TRACE_LOG( + "RecvFrom called on fd:|%s|. Current read buf length:|%d|", GetName(), + GRPC_SLICE_LENGTH(read_buf_)); + if (GRPC_SLICE_LENGTH(read_buf_) == 0) { + WSASetLastError(WSAEWOULDBLOCK); + return -1; + } + ares_ssize_t bytes_read = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(read_buf_) && i < data_len; i++) { + ((char*)data)[i] = GRPC_SLICE_START_PTR(read_buf_)[i]; + bytes_read++; + } + read_buf_ = grpc_slice_sub_no_ref(read_buf_, bytes_read, + GRPC_SLICE_LENGTH(read_buf_)); + /* c-ares overloads this recv_from virtual socket function to receive + * data on both UDP and TCP sockets, and from is nullptr for TCP. */ + if (from != nullptr) { + GPR_ASSERT(*from_len <= recv_from_source_addr_len_); + memcpy(from, &recv_from_source_addr_, recv_from_source_addr_len_); + *from_len = recv_from_source_addr_len_; + } + return bytes_read; + } + + grpc_slice FlattenIovec(const struct iovec* iov, int iov_count) { + int total = 0; + for (int i = 0; i < iov_count; i++) { + total += iov[i].iov_len; + } + grpc_slice out = GRPC_SLICE_MALLOC(total); + size_t cur = 0; + for (int i = 0; i < iov_count; i++) { + for (int k = 0; k < iov[i].iov_len; k++) { + GRPC_SLICE_START_PTR(out)[cur++] = ((char*)iov[i].iov_base)[k]; + } + } + return out; + } + + int SendWriteBuf(LPDWORD bytes_sent_ptr, LPWSAOVERLAPPED overlapped) { + WSABUF buf; + buf.len = GRPC_SLICE_LENGTH(write_buf_); + buf.buf = (char*)GRPC_SLICE_START_PTR(write_buf_); + DWORD flags = 0; + int out = WSASend(grpc_winsocket_wrapped_socket(winsocket_), &buf, 1, + bytes_sent_ptr, flags, overlapped, nullptr); + GRPC_CARES_TRACE_LOG( + "WSASend: name:%s. buf len:%d. bytes sent: %d. overlapped %p. return " + "val: %d", + GetName(), buf.len, *bytes_sent_ptr, overlapped, out); + return out; + } + + ares_ssize_t TrySendWriteBufSyncNonBlocking() { + GPR_ASSERT(write_state_ == WRITE_IDLE); + ares_ssize_t total_sent; + DWORD bytes_sent = 0; + if (SendWriteBuf(&bytes_sent, nullptr) != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG( + "TrySendWriteBufSyncNonBlocking: SendWriteBuf error:|%s|. fd:|%s|", + msg, GetName()); + gpr_free(msg); + if (WSAGetLastError() == WSA_IO_PENDING) { + WSASetLastError(WSAEWOULDBLOCK); + write_state_ = WRITE_REQUESTED; + } + } + write_buf_ = grpc_slice_sub_no_ref(write_buf_, bytes_sent, + GRPC_SLICE_LENGTH(write_buf_)); + return bytes_sent; + } + + ares_ssize_t SendV(const struct iovec* iov, int iov_count) { + GRPC_CARES_TRACE_LOG("SendV called on fd:|%s|. Current write state: %d", + GetName(), write_state_); + switch (write_state_) { + case WRITE_IDLE: + GPR_ASSERT(GRPC_SLICE_LENGTH(write_buf_) == 0); + grpc_slice_unref_internal(write_buf_); + write_buf_ = FlattenIovec(iov, iov_count); + return TrySendWriteBufSyncNonBlocking(); + case WRITE_REQUESTED: + case WRITE_PENDING: + WSASetLastError(WSAEWOULDBLOCK); + return -1; + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + grpc_slice currently_attempted = FlattenIovec(iov, iov_count); + GPR_ASSERT(GRPC_SLICE_LENGTH(currently_attempted) >= + GRPC_SLICE_LENGTH(write_buf_)); + ares_ssize_t total_sent = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(write_buf_); i++) { + GPR_ASSERT(GRPC_SLICE_START_PTR(currently_attempted)[i] == + GRPC_SLICE_START_PTR(write_buf_)[i]); + total_sent++; + } + grpc_slice_unref_internal(write_buf_); + write_buf_ = + grpc_slice_sub_no_ref(currently_attempted, total_sent, + GRPC_SLICE_LENGTH(currently_attempted)); + write_state_ = WRITE_IDLE; + total_sent += TrySendWriteBufSyncNonBlocking(); + return total_sent; + } abort(); } - bool IsFdStillReadableLocked() override { abort(); } - void ShutdownLocked(grpc_error* error) override { abort(); } - ares_socket_t GetWrappedAresSocketLocked() override { abort(); } - const char* GetName() override { abort(); } + + int Connect(const struct sockaddr* target, ares_socklen_t target_len) { + SOCKET s = grpc_winsocket_wrapped_socket(winsocket_); + GRPC_CARES_TRACE_LOG("Connect: fd:|%s|", GetName()); + int out = + WSAConnect(s, target, target_len, nullptr, nullptr, nullptr, nullptr); + if (out != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG("Connect error code:|%d|, msg:|%s|. fd:|%s|", + WSAGetLastError(), msg, GetName()); + gpr_free(msg); + // c-ares expects a posix-style connect API + out = -1; + } + return out; + } + + static void OnIocpReadable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpReadableInner(error); + } + + void OnIocpReadableInner(grpc_error* error) { + if (error == GRPC_ERROR_NONE) { + if (winsocket_->read_info.wsa_error != 0) { + /* WSAEMSGSIZE would be due to receiving more data + * than our read buffer's fixed capacity. Assume that + * the connection is TCP and read the leftovers + * in subsequent c-ares reads. */ + if (winsocket_->read_info.wsa_error != WSAEMSGSIZE) { + GRPC_ERROR_UNREF(error); + char* msg = gpr_format_message(winsocket_->read_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpReadableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + } + if (error == GRPC_ERROR_NONE) { + read_buf_ = grpc_slice_sub_no_ref(read_buf_, 0, + winsocket_->read_info.bytes_transfered); + } else { + grpc_slice_unref_internal(read_buf_); + read_buf_ = grpc_empty_slice(); + } + GRPC_CARES_TRACE_LOG( + "OnIocpReadable finishing. read buf length now:|%d|. :fd:|%s|", + GRPC_SLICE_LENGTH(read_buf_), GetName()); + ScheduleAndNullReadClosure(error); + } + + static void OnIocpWriteable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpWriteableInner(error); + } + + void OnIocpWriteableInner(grpc_error* error) { + GRPC_CARES_TRACE_LOG("OnIocpWriteableInner. fd:|%s|", GetName()); + if (error == GRPC_ERROR_NONE) { + if (winsocket_->write_info.wsa_error != 0) { + char* msg = gpr_format_message(winsocket_->write_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpWriteableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + GRPC_ERROR_UNREF(error); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + GPR_ASSERT(write_state_ == WRITE_PENDING); + if (error == GRPC_ERROR_NONE) { + write_state_ = WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY; + write_buf_ = grpc_slice_sub_no_ref( + write_buf_, 0, winsocket_->write_info.bytes_transfered); + } else { + grpc_slice_unref_internal(write_buf_); + write_buf_ = grpc_empty_slice(); + } + ScheduleAndNullWriteClosure(error); + } + + bool gotten_into_driver_list() const { return gotten_into_driver_list_; } + void set_gotten_into_driver_list() { gotten_into_driver_list_ = true; } + + grpc_combiner* combiner_; + char recv_from_source_addr_[200]; + ares_socklen_t recv_from_source_addr_len_; + grpc_slice read_buf_; + grpc_slice write_buf_; + grpc_closure* read_closure_ = nullptr; + grpc_closure* write_closure_ = nullptr; + grpc_closure outer_read_closure_; + grpc_closure outer_write_closure_; + grpc_winsocket* winsocket_; + WriteState write_state_; + char* name_ = nullptr; + bool gotten_into_driver_list_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return nullptr; -} +struct SockToPolledFdEntry { + SockToPolledFdEntry(SOCKET s, GrpcPolledFdWindows* fd) + : socket(s), polled_fd(fd) {} + SOCKET socket; + GrpcPolledFdWindows* polled_fd; + SockToPolledFdEntry* next = nullptr; +}; + +/* A SockToPolledFdMap can make ares_socket_t types (SOCKET's on windows) + * to GrpcPolledFdWindow's, and is used to find the appropriate + * GrpcPolledFdWindows to handle a virtual socket call when c-ares makes that + * socket call on the ares_socket_t type. Instances are owned by and one-to-one + * with a GrpcPolledFdWindows factory and event driver */ +class SockToPolledFdMap { + public: + SockToPolledFdMap(grpc_combiner* combiner) { + combiner_ = GRPC_COMBINER_REF(combiner, "sock to polled fd map"); + } + + ~SockToPolledFdMap() { + GPR_ASSERT(head_ == nullptr); + GRPC_COMBINER_UNREF(combiner_, "sock to polled fd map"); + } + + void AddNewSocket(SOCKET s, GrpcPolledFdWindows* polled_fd) { + SockToPolledFdEntry* new_node = New(s, polled_fd); + new_node->next = head_; + head_ = new_node; + } + + GrpcPolledFdWindows* LookupPolledFd(SOCKET s) { + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + GPR_ASSERT(node->polled_fd != nullptr); + return node->polled_fd; + } + } + abort(); + } + + void RemoveEntry(SOCKET s) { + GPR_ASSERT(head_ != nullptr); + SockToPolledFdEntry** prev = &head_; + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + *prev = node->next; + Delete(node); + return; + } + prev = &node->next; + } + abort(); + } + + /* These virtual socket functions are called from within the c-ares + * library. These methods generally dispatch those socket calls to the + * appropriate methods. The virtual "socket" and "close" methods are + * special and instead create/add and remove/destroy GrpcPolledFdWindows + * objects. + */ + static ares_socket_t Socket(int af, int type, int protocol, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + SOCKET s = WSASocket(af, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (s == INVALID_SOCKET) { + return s; + } + grpc_tcp_set_non_block(s); + GrpcPolledFdWindows* polled_fd = + New(s, map->combiner_); + map->AddNewSocket(s, polled_fd); + return s; + } + + static int Connect(ares_socket_t as, const struct sockaddr* target, + ares_socklen_t target_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->Connect(target, target_len); + } + + static ares_ssize_t SendV(ares_socket_t as, const struct iovec* iov, + int iovec_count, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->SendV(iov, iovec_count); + } + + static ares_ssize_t RecvFrom(ares_socket_t as, void* data, size_t data_len, + int flags, struct sockaddr* from, + ares_socklen_t* from_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->RecvFrom(data, data_len, flags, from, from_len); + } + + static int CloseSocket(SOCKET s, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(s); + map->RemoveEntry(s); + // If a gRPC polled fd has not made it in to the driver's list yet, then + // the driver has not and will never see this socket. + if (!polled_fd->gotten_into_driver_list()) { + polled_fd->ShutdownLocked(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Shut down c-ares fd before without it ever having made it into the " + "driver's list")); + return 0; + } + return 0; + } + + private: + SockToPolledFdEntry* head_ = nullptr; + grpc_combiner* combiner_; +}; + +const struct ares_socket_functions custom_ares_sock_funcs = { + &SockToPolledFdMap::Socket /* socket */, + &SockToPolledFdMap::CloseSocket /* close */, + &SockToPolledFdMap::Connect /* connect */, + &SockToPolledFdMap::RecvFrom /* recvfrom */, + &SockToPolledFdMap::SendV /* sendv */, +}; + +class GrpcPolledFdFactoryWindows : public GrpcPolledFdFactory { + public: + GrpcPolledFdFactoryWindows(grpc_combiner* combiner) + : sock_to_polled_fd_map_(combiner) {} + + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + GrpcPolledFdWindows* polled_fd = sock_to_polled_fd_map_.LookupPolledFd(as); + // Set a flag so that the virtual socket "close" method knows it + // doesn't need to call ShutdownLocked, since now the driver will. + polled_fd->set_gotten_into_driver_list(); + return polled_fd; + } -void ConfigureAresChannelLocked(ares_channel* channel) { abort(); } + void ConfigureAresChannelLocked(ares_channel channel) override { + ares_set_socket_functions(channel, &custom_ares_sock_funcs, + &sock_to_polled_fd_map_); + } + + private: + SockToPolledFdMap sock_to_polled_fd_map_; +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr( + New(combiner)); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index b3d6437e9a6..485998f5e44 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -49,6 +49,8 @@ static gpr_mu g_init_mu; grpc_core::TraceFlag grpc_trace_cares_address_sorting(false, "cares_address_sorting"); +grpc_core::TraceFlag grpc_trace_cares_resolver(false, "cares_resolver"); + struct grpc_ares_request { /** indicates the DNS server to use, if specified */ struct ares_addr_port_node dns_server_addr; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index 17eaa7ccf06..ca5779e1d79 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -28,6 +28,13 @@ extern grpc_core::TraceFlag grpc_trace_cares_address_sorting; +extern grpc_core::TraceFlag grpc_trace_cares_resolver; + +#define GRPC_CARES_TRACE_LOG(format, ...) \ + if (grpc_trace_cares_resolver.enabled()) { \ + gpr_log(GPR_DEBUG, "(c-ares resolver) " format, __VA_ARGS__); \ + } + typedef struct grpc_ares_request grpc_ares_request; /* Asynchronously resolve \a name. Use \a default_port if a port isn't diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc index fae4c33a178..282caf215c7 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc @@ -58,6 +58,8 @@ class NativeDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -158,6 +160,13 @@ void NativeDnsResolver::RequestReresolutionLocked() { } } +void NativeDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void NativeDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index abf3a039f36..4a45ca081c4 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -132,6 +132,8 @@ struct grpc_subchannel { bool have_alarm; /** have we started the backoff loop */ bool backoff_begun; + // reset_backoff() was called while alarm was pending + bool deferred_reset_backoff; /** our alarm */ grpc_timer alarm; @@ -453,6 +455,9 @@ static void on_alarm(void* arg, grpc_error* error) { if (c->disconnected) { error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected", &error, 1); + } else if (c->deferred_reset_backoff) { + c->deferred_reset_backoff = false; + error = GRPC_ERROR_NONE; } else { GRPC_ERROR_REF(error); } @@ -690,6 +695,19 @@ static void on_subchannel_connected(void* arg, grpc_error* error) { grpc_channel_args_destroy(delete_channel_args); } +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel) { + gpr_mu_lock(&subchannel->mu); + if (subchannel->have_alarm) { + subchannel->deferred_reset_backoff = true; + grpc_timer_cancel(&subchannel->alarm); + } else { + subchannel->backoff_begun = false; + subchannel->backoff->Reset(); + maybe_start_connecting_locked(subchannel); + } + gpr_mu_unlock(&subchannel->mu); +} + /* * grpc_subchannel_call implementation */ diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index e3d9296c262..84febb52040 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -152,6 +152,13 @@ grpc_subchannel_get_connected_subchannel(grpc_subchannel* c); const grpc_subchannel_key* grpc_subchannel_get_key( const grpc_subchannel* subchannel); +// Resets the connection backoff of the subchannel. +// TODO(roth): Move connection backoff out of subchannels and up into LB +// policy code (probably by adding a SubchannelGroup between +// SubchannelList and SubchannelData), at which point this method can +// go away. +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel); + /** continue processing a transport op */ void grpc_subchannel_call_process_op(grpc_subchannel_call* subchannel_call, grpc_transport_stream_op_batch* op); diff --git a/src/core/ext/filters/deadline/deadline_filter.cc b/src/core/ext/filters/deadline/deadline_filter.cc index d23ad67ad51..3bd30593122 100644 --- a/src/core/ext/filters/deadline/deadline_filter.cc +++ b/src/core/ext/filters/deadline/deadline_filter.cc @@ -379,10 +379,10 @@ static bool maybe_add_deadline_filter(grpc_channel_stack_builder* builder, void grpc_deadline_filter_init(void) { grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, maybe_add_deadline_filter, (void*)&grpc_client_deadline_filter); grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, maybe_add_deadline_filter, (void*)&grpc_server_deadline_filter); } diff --git a/src/core/ext/filters/http/client_authority_filter.cc b/src/core/ext/filters/http/client_authority_filter.cc index ddc939ed121..3c0ae47e8d5 100644 --- a/src/core/ext/filters/http/client_authority_filter.cc +++ b/src/core/ext/filters/http/client_authority_filter.cc @@ -146,12 +146,12 @@ static bool add_client_authority_filter(grpc_channel_stack_builder* builder, } void grpc_client_authority_filter_init(void) { - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, - add_client_authority_filter, - (void*)&grpc_client_authority_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - add_client_authority_filter, - (void*)&grpc_client_authority_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + add_client_authority_filter, (void*)&grpc_client_authority_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + add_client_authority_filter, (void*)&grpc_client_authority_filter); } void grpc_client_authority_filter_shutdown(void) {} diff --git a/src/core/ext/filters/http/http_filters_plugin.cc b/src/core/ext/filters/http/http_filters_plugin.cc index f03fa0141df..38757710f39 100644 --- a/src/core/ext/filters/http/http_filters_plugin.cc +++ b/src/core/ext/filters/http/http_filters_plugin.cc @@ -18,6 +18,7 @@ #include +#include #include #include "src/core/ext/filters/http/client/http_client_filter.h" @@ -51,15 +52,15 @@ static bool maybe_add_optional_filter(grpc_channel_stack_builder* builder, bool enable = grpc_channel_arg_get_bool( grpc_channel_args_find(channel_args, filtarg->control_channel_arg), !grpc_channel_args_want_minimal_stack(channel_args)); - return enable ? grpc_channel_stack_builder_prepend_filter( + return enable ? grpc_channel_stack_builder_append_filter( builder, filtarg->filter, nullptr, nullptr) : true; } -static bool maybe_add_required_filter(grpc_channel_stack_builder* builder, - void* arg) { +static bool maybe_append_required_filter(grpc_channel_stack_builder* builder, + void* arg) { return is_building_http_like_transport(builder) - ? grpc_channel_stack_builder_prepend_filter( + ? grpc_channel_stack_builder_append_filter( builder, static_cast(arg), nullptr, nullptr) : true; @@ -67,23 +68,23 @@ static bool maybe_add_required_filter(grpc_channel_stack_builder* builder, void grpc_http_filters_init(void) { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage( - GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_client_filter); + GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_client_filter); grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_client_filter); + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_client_filter); grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_server_filter); + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_server_filter); } void grpc_http_filters_shutdown(void) {} diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc index 6529046a5e7..0c4ffea27bf 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc @@ -345,7 +345,8 @@ struct ServerLoadReportingFilterStaticRegistrar { if (registered) return; RegisterChannelFilter( - "server_load_reporting", GRPC_SERVER_CHANNEL, INT_MAX, + "server_load_reporting", GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_LOW, true, MaybeAddServerLoadReportingFilter); // Access measures to ensure they are initialized. Otherwise, we can't // create any valid view before the first RPC. diff --git a/src/core/ext/filters/max_age/max_age_filter.cc b/src/core/ext/filters/max_age/max_age_filter.cc index 1fe8288bd0a..7db30d5b488 100644 --- a/src/core/ext/filters/max_age/max_age_filter.cc +++ b/src/core/ext/filters/max_age/max_age_filter.cc @@ -536,7 +536,7 @@ static bool maybe_add_max_age_filter(grpc_channel_stack_builder* builder, void grpc_max_age_filter_init(void) { grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_max_age_filter, nullptr); } diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index c7fc3f2e627..1bd9cf1426b 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -311,13 +311,13 @@ static bool maybe_add_message_size_filter(grpc_channel_stack_builder* builder, void grpc_message_size_filter_init(void) { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); } diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc index bd6fec6fbe2..bc6fa0d0eb0 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.cc +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -813,11 +813,7 @@ static void set_write_state(grpc_chttp2_transport* t, write_state_name(st), reason)); t->write_state = st; if (st == GRPC_CHTTP2_WRITE_STATE_IDLE) { - grpc_chttp2_stream* s; - while (grpc_chttp2_list_pop_waiting_for_write_stream(t, &s)) { - GRPC_CLOSURE_LIST_SCHED(&s->run_after_write); - GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:write_closure_sched"); - } + GRPC_CLOSURE_LIST_SCHED(&t->run_after_write); if (t->close_transport_on_writes_finished != nullptr) { grpc_error* err = t->close_transport_on_writes_finished; t->close_transport_on_writes_finished = nullptr; @@ -1208,14 +1204,11 @@ void grpc_chttp2_complete_closure_step(grpc_chttp2_transport* t, grpc_error_add_child(closure->error_data.error, error); } if (closure->next_data.scratch < CLOSURE_BARRIER_FIRST_REF_BIT) { - if (s->seen_error || (t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || + if ((t->write_state == GRPC_CHTTP2_WRITE_STATE_IDLE) || !(closure->next_data.scratch & CLOSURE_BARRIER_MAY_COVER_WRITE)) { GRPC_CLOSURE_RUN(closure, closure->error_data.error); } else { - if (grpc_chttp2_list_add_waiting_for_write_stream(t, s)) { - GRPC_CHTTP2_STREAM_REF(s, "chttp2:pending_write_closure"); - } - grpc_closure_list_append(&s->run_after_write, closure, + grpc_closure_list_append(&t->run_after_write, closure, closure->error_data.error); } } @@ -2016,10 +2009,6 @@ static void remove_stream(grpc_chttp2_transport* t, uint32_t id, void grpc_chttp2_cancel_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s, grpc_error* due_to_error) { - GRPC_CLOSURE_LIST_SCHED(&s->run_after_write); - if (grpc_chttp2_list_remove_waiting_for_write_stream(t, s)) { - GRPC_CHTTP2_STREAM_UNREF(s, "chttp2:pending_write_closure"); - } if (!t->is_client && !s->sent_trailing_metadata && grpc_error_has_clear_grpc_status(due_to_error)) { close_from_api(t, s, due_to_error); diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 4f1a08d98b5..ca6e7159787 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -54,8 +54,6 @@ typedef enum { /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, - /** streams with closures waiting to be run on a write **/ - GRPC_CHTTP2_LIST_WAITING_FOR_WRITE, STREAM_LIST_COUNT /* must be last */ } grpc_chttp2_stream_list_id; @@ -433,6 +431,9 @@ struct grpc_chttp2_transport { */ grpc_error* close_transport_on_writes_finished; + /* a list of closures to run after writes are finished */ + grpc_closure_list run_after_write; + /* buffer pool state */ /** have we scheduled a benign cleanup? */ bool benign_reclaimer_registered; @@ -583,7 +584,6 @@ struct grpc_chttp2_stream { grpc_slice_buffer flow_controlled_buffer; - grpc_closure_list run_after_write; grpc_chttp2_write_cb* on_flow_controlled_cbs; grpc_chttp2_write_cb* on_write_finished_cbs; grpc_chttp2_write_cb* finish_after_write; @@ -686,13 +686,6 @@ bool grpc_chttp2_list_pop_stalled_by_stream(grpc_chttp2_transport* t, bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s); -bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s); -bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream** s); -bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s); - /********* Flow Control ***************/ // Takes in a flow control action and performs all the needed operations. diff --git a/src/core/ext/transport/chttp2/transport/stream_lists.cc b/src/core/ext/transport/chttp2/transport/stream_lists.cc index 50bfe36a863..6626170a7e4 100644 --- a/src/core/ext/transport/chttp2/transport/stream_lists.cc +++ b/src/core/ext/transport/chttp2/transport/stream_lists.cc @@ -35,8 +35,6 @@ static const char* stream_list_id_string(grpc_chttp2_stream_list_id id) { return "stalled_by_stream"; case GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY: return "waiting_for_concurrency"; - case GRPC_CHTTP2_LIST_WAITING_FOR_WRITE: - return "waiting_for_write"; case STREAM_LIST_COUNT: GPR_UNREACHABLE_CODE(return "unknown"); } @@ -216,18 +214,3 @@ bool grpc_chttp2_list_remove_stalled_by_stream(grpc_chttp2_transport* t, grpc_chttp2_stream* s) { return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_STALLED_BY_STREAM); } - -bool grpc_chttp2_list_add_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - return stream_list_add(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); -} - -bool grpc_chttp2_list_pop_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream** s) { - return stream_list_pop(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); -} - -bool grpc_chttp2_list_remove_waiting_for_write_stream(grpc_chttp2_transport* t, - grpc_chttp2_stream* s) { - return stream_list_maybe_remove(t, s, GRPC_CHTTP2_LIST_WAITING_FOR_WRITE); -} diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index 90a02546633..45be90af5fb 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -230,8 +230,8 @@ static void bind_transport(grpc_channel_stack* channel_stack, grpc_transport_stream_size(static_cast(t)); } -bool grpc_add_connected_filter(grpc_channel_stack_builder* builder, - void* arg_must_be_null) { +bool grpc_append_connected_filter(grpc_channel_stack_builder* builder, + void* arg_must_be_null) { GPR_ASSERT(arg_must_be_null == nullptr); grpc_transport* t = grpc_channel_stack_builder_get_transport(builder); GPR_ASSERT(t != nullptr); diff --git a/src/core/lib/channel/connected_channel.h b/src/core/lib/channel/connected_channel.h index faa1c73a21f..280daf040da 100644 --- a/src/core/lib/channel/connected_channel.h +++ b/src/core/lib/channel/connected_channel.h @@ -25,8 +25,8 @@ extern const grpc_channel_filter grpc_connected_filter; -bool grpc_add_connected_filter(grpc_channel_stack_builder* builder, - void* arg_must_be_null); +bool grpc_append_connected_filter(grpc_channel_stack_builder* builder, + void* arg_must_be_null); /* Debug helper to dig the transport stream out of a call element */ grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem); diff --git a/src/core/lib/iomgr/call_combiner.h b/src/core/lib/iomgr/call_combiner.h index 641fa180820..6f7ddd40435 100644 --- a/src/core/lib/iomgr/call_combiner.h +++ b/src/core/lib/iomgr/call_combiner.h @@ -102,7 +102,10 @@ void grpc_call_combiner_stop(grpc_call_combiner* call_combiner, /// If \a closure is NULL, then no closure will be invoked on /// cancellation; this effectively unregisters the previously set closure. /// However, most filters will not need to explicitly unregister their -/// callbacks, as this is done automatically when the call is destroyed. +/// callbacks, as this is done automatically when the call is destroyed. Filters +/// that schedule the cancellation closure on ExecCtx do not need to take a ref +/// on the call stack to guarantee closure liveness. This is done by explicitly +/// flushing ExecCtx after the unregistration during call destruction. void grpc_call_combiner_set_notify_on_cancel(grpc_call_combiner* call_combiner, grpc_closure* closure); diff --git a/src/core/lib/iomgr/iocp_windows.cc b/src/core/lib/iomgr/iocp_windows.cc index ce772310362..ad325fe2156 100644 --- a/src/core/lib/iomgr/iocp_windows.cc +++ b/src/core/lib/iomgr/iocp_windows.cc @@ -89,10 +89,15 @@ grpc_iocp_work_status grpc_iocp_work(grpc_millis deadline) { } else { abort(); } - success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, - FALSE, &flags); - info->bytes_transfered = bytes; - info->wsa_error = success ? 0 : WSAGetLastError(); + if (socket->shutdown_called) { + info->bytes_transfered = 0; + info->wsa_error = WSA_OPERATION_ABORTED; + } else { + success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, + FALSE, &flags); + info->bytes_transfered = bytes; + info->wsa_error = success ? 0 : WSAGetLastError(); + } GPR_ASSERT(overlapped == &info->overlapped); grpc_socket_become_ready(socket, info); return GRPC_IOCP_WORK_WORK; diff --git a/src/core/lib/iomgr/socket_windows.cc b/src/core/lib/iomgr/socket_windows.cc index 4ad31cb35d1..999c6646ad4 100644 --- a/src/core/lib/iomgr/socket_windows.cc +++ b/src/core/lib/iomgr/socket_windows.cc @@ -52,6 +52,10 @@ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name) { return r; } +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket) { + return socket->socket; +} + /* Schedule a shutdown of the socket operations. Will call the pending operations to abort them. We need to do that this way because of the various callsites of that function, which happens to be in various diff --git a/src/core/lib/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h index b09b9da5628..46d7d583560 100644 --- a/src/core/lib/iomgr/socket_windows.h +++ b/src/core/lib/iomgr/socket_windows.h @@ -92,6 +92,8 @@ typedef struct grpc_winsocket { it will be responsible for closing it. */ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name); +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket); + /* Initiate an asynchronous shutdown of the socket. Will call off any pending operation to cancel them. */ void grpc_winsocket_shutdown(grpc_winsocket* socket); diff --git a/src/core/lib/iomgr/tcp_windows.cc b/src/core/lib/iomgr/tcp_windows.cc index 5d316d477b7..b3cb442f18f 100644 --- a/src/core/lib/iomgr/tcp_windows.cc +++ b/src/core/lib/iomgr/tcp_windows.cc @@ -53,7 +53,7 @@ extern grpc_core::TraceFlag grpc_tcp_trace; -static grpc_error* set_non_block(SOCKET sock) { +grpc_error* grpc_tcp_set_non_block(SOCKET sock) { int status; uint32_t param = 1; DWORD ret; @@ -90,7 +90,7 @@ static grpc_error* enable_loopback_fast_path(SOCKET sock) { grpc_error* grpc_tcp_prepare_socket(SOCKET sock) { grpc_error* err; - err = set_non_block(sock); + err = grpc_tcp_set_non_block(sock); if (err != GRPC_ERROR_NONE) return err; err = set_dualstack(sock); if (err != GRPC_ERROR_NONE) return err; diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 161a545a2a5..04ef8102b65 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -46,6 +46,8 @@ grpc_endpoint* grpc_tcp_create(grpc_winsocket* socket, grpc_error* grpc_tcp_prepare_socket(SOCKET sock); +grpc_error* grpc_tcp_set_non_block(SOCKET sock); + #endif #endif /* GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H */ diff --git a/src/core/lib/security/security_connector/load_system_roots.h b/src/core/lib/security/security_connector/load_system_roots.h new file mode 100644 index 00000000000..5fdec154988 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots.h @@ -0,0 +1,29 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H +#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H + +namespace grpc_core { + +// Returns a slice containing roots from the OS trust store +grpc_slice LoadSystemRootCerts(); + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_H */ diff --git a/src/core/lib/security/security_connector/load_system_roots_fallback.cc b/src/core/lib/security/security_connector/load_system_roots_fallback.cc new file mode 100644 index 00000000000..73d1245f331 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_fallback.cc @@ -0,0 +1,32 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include "src/core/lib/security/security_connector/load_system_roots.h" + +#ifndef GPR_LINUX + +namespace grpc_core { + +grpc_slice LoadSystemRootCerts() { return grpc_empty_slice(); } + +} // namespace grpc_core + +#endif /* GPR_LINUX */ diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.cc b/src/core/lib/security/security_connector/load_system_roots_linux.cc new file mode 100644 index 00000000000..924fa8a3e26 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_linux.cc @@ -0,0 +1,165 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include + +#include +#include "src/core/lib/security/security_connector/load_system_roots_linux.h" + +#ifdef GPR_LINUX + +#include "src/core/lib/security/security_connector/load_system_roots.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/inlined_vector.h" +#include "src/core/lib/iomgr/load_file.h" + +namespace grpc_core { +namespace { + +const char* kLinuxCertFiles[] = { + "/etc/ssl/certs/ca-certificates.crt", "/etc/pki/tls/certs/ca-bundle.crt", + "/etc/ssl/ca-bundle.pem", "/etc/pki/tls/cacert.pem", + "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"}; +const char* kLinuxCertDirectories[] = { + "/etc/ssl/certs", "/system/etc/security/cacerts", "/usr/local/share/certs", + "/etc/pki/tls/certs", "/etc/openssl/certs"}; + +grpc_slice GetSystemRootCerts() { + grpc_slice valid_bundle_slice = grpc_empty_slice(); + size_t num_cert_files_ = GPR_ARRAY_SIZE(kLinuxCertFiles); + for (size_t i = 0; i < num_cert_files_; i++) { + grpc_error* error = + grpc_load_file(kLinuxCertFiles[i], 1, &valid_bundle_slice); + if (error == GRPC_ERROR_NONE) { + return valid_bundle_slice; + } + } + return grpc_empty_slice(); +} + +} // namespace + +void GetAbsoluteFilePath(const char* valid_file_dir, + const char* file_entry_name, char* path_buffer) { + if (valid_file_dir != nullptr && file_entry_name != nullptr) { + int path_len = snprintf(path_buffer, MAXPATHLEN, "%s/%s", valid_file_dir, + file_entry_name); + if (path_len == 0) { + gpr_log(GPR_ERROR, "failed to get absolute path for file: %s", + file_entry_name); + } + } +} + +grpc_slice CreateRootCertsBundle(const char* certs_directory) { + grpc_slice bundle_slice = grpc_empty_slice(); + if (certs_directory == nullptr) { + return bundle_slice; + } + DIR* ca_directory = opendir(certs_directory); + if (ca_directory == nullptr) { + return bundle_slice; + } + struct FileData { + char path[MAXPATHLEN]; + off_t size; + }; + InlinedVector roots_filenames; + size_t total_bundle_size = 0; + struct dirent* directory_entry; + while ((directory_entry = readdir(ca_directory)) != nullptr) { + struct stat dir_entry_stat; + const char* file_entry_name = directory_entry->d_name; + FileData file_data; + GetAbsoluteFilePath(certs_directory, file_entry_name, file_data.path); + int stat_return = stat(file_data.path, &dir_entry_stat); + if (stat_return == -1 || !S_ISREG(dir_entry_stat.st_mode)) { + // no subdirectories. + if (stat_return == -1) { + gpr_log(GPR_ERROR, "failed to get status for file: %s", file_data.path); + } + continue; + } + file_data.size = dir_entry_stat.st_size; + total_bundle_size += file_data.size; + roots_filenames.push_back(file_data); + } + closedir(ca_directory); + char* bundle_string = static_cast(gpr_zalloc(total_bundle_size + 1)); + size_t bytes_read = 0; + for (size_t i = 0; i < roots_filenames.size(); i++) { + int file_descriptor = open(roots_filenames[i].path, O_RDONLY); + if (file_descriptor != -1) { + // Read file into bundle. + size_t cert_file_size = roots_filenames[i].size; + int read_ret = + read(file_descriptor, bundle_string + bytes_read, cert_file_size); + if (read_ret != -1) { + bytes_read += read_ret; + } else { + gpr_log(GPR_ERROR, "failed to read file: %s", roots_filenames[i].path); + } + } + } + bundle_slice = grpc_slice_new(bundle_string, bytes_read, gpr_free); + return bundle_slice; +} + +grpc_slice LoadSystemRootCerts() { + grpc_slice result = grpc_empty_slice(); + // Prioritize user-specified custom directory if flag is set. + char* custom_dir = gpr_getenv("GRPC_SYSTEM_SSL_ROOTS_DIR"); + if (custom_dir != nullptr) { + result = CreateRootCertsBundle(custom_dir); + gpr_free(custom_dir); + } + // If the custom directory is empty/invalid/not specified, fallback to + // distribution-specific directory. + if (GRPC_SLICE_IS_EMPTY(result)) { + result = GetSystemRootCerts(); + } + if (GRPC_SLICE_IS_EMPTY(result)) { + for (size_t i = 0; i < GPR_ARRAY_SIZE(kLinuxCertDirectories); i++) { + result = CreateRootCertsBundle(kLinuxCertDirectories[i]); + if (!GRPC_SLICE_IS_EMPTY(result)) { + break; + } + } + } + return result; +} + +} // namespace grpc_core + +#endif /* GPR_LINUX */ diff --git a/src/core/lib/security/security_connector/load_system_roots_linux.h b/src/core/lib/security/security_connector/load_system_roots_linux.h new file mode 100644 index 00000000000..12617df4928 --- /dev/null +++ b/src/core/lib/security/security_connector/load_system_roots_linux.h @@ -0,0 +1,44 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H +#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H + +#include + +#ifdef GPR_LINUX + +namespace grpc_core { + +// Creates a bundle slice containing the contents of all certificate files in +// a directory. +// Returns such slice. +// Exposed for testing purposes only. +grpc_slice CreateRootCertsBundle(const char* certs_directory); + +// Gets the absolute file path needed to load a certificate file. +// Populates path_buffer, which must be of size MAXPATHLEN. +// Exposed for testing purposes only. +void GetAbsoluteFilePath(const char* valid_file_dir, + const char* file_entry_name, char* path_buffer); + +} // namespace grpc_core + +#endif /* GPR_LINUX */ +#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_LOAD_SYSTEM_ROOTS_LINUX_H \ + */ diff --git a/src/core/lib/security/security_connector/security_connector.cc b/src/core/lib/security/security_connector/security_connector.cc index 59cf3a0af18..04b4c87c713 100644 --- a/src/core/lib/security/security_connector/security_connector.cc +++ b/src/core/lib/security/security_connector/security_connector.cc @@ -21,7 +21,6 @@ #include "src/core/lib/security/security_connector/security_connector.h" #include -#include #include #include @@ -39,6 +38,7 @@ #include "src/core/lib/security/credentials/credentials.h" #include "src/core/lib/security/credentials/fake/fake_credentials.h" #include "src/core/lib/security/credentials/ssl/ssl_credentials.h" +#include "src/core/lib/security/security_connector/load_system_roots.h" #include "src/core/lib/security/transport/secure_endpoint.h" #include "src/core/lib/security/transport/security_handshaker.h" #include "src/core/lib/security/transport/target_authority_table.h" @@ -57,6 +57,12 @@ static const char* installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem"; #endif +/** Environment variable used as a flag to enable/disable loading system root + certificates from the OS trust store. */ +#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#endif + #ifndef TSI_OPENSSL_ALPN_SUPPORT #define TSI_OPENSSL_ALPN_SUPPORT 1 #endif @@ -1186,6 +1192,10 @@ const char* DefaultSslRootStore::GetPemRootCerts() { grpc_slice DefaultSslRootStore::ComputePemRootCerts() { grpc_slice result = grpc_empty_slice(); + char* use_system_roots_env_value = + gpr_getenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + const bool use_system_roots = gpr_is_true(use_system_roots_env_value); + gpr_free(use_system_roots_env_value); // First try to load the roots from the environment. char* default_root_certs_path = gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); @@ -1207,7 +1217,11 @@ grpc_slice DefaultSslRootStore::ComputePemRootCerts() { } gpr_free(pem_root_certs); } - // Fall back to installed certs if needed. + // Try loading roots from OS trust store if flag is enabled. + if (GRPC_SLICE_IS_EMPTY(result) && use_system_roots) { + result = LoadSystemRootCerts(); + } + // Fallback to roots manually shipped with gRPC. if (GRPC_SLICE_IS_EMPTY(result) && ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { GRPC_LOG_IF_ERROR("load_file", diff --git a/src/core/lib/security/transport/client_auth_filter.cc b/src/core/lib/security/transport/client_auth_filter.cc index 9b5c6f34906..0f125e7c26d 100644 --- a/src/core/lib/security/transport/client_auth_filter.cc +++ b/src/core/lib/security/transport/client_auth_filter.cc @@ -167,7 +167,6 @@ static void cancel_get_request_metadata(void* arg, grpc_error* error) { grpc_call_credentials_cancel_get_request_metadata( calld->creds, &calld->md_array, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_get_request_metadata"); } static void send_security_metadata(grpc_call_element* elem, @@ -222,7 +221,6 @@ static void send_security_metadata(grpc_call_element* elem, GRPC_ERROR_UNREF(error); } else { // Async return; register cancellation closure with call combiner. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_get_request_metadata"); grpc_call_combiner_set_notify_on_cancel( calld->call_combiner, GRPC_CLOSURE_INIT(&calld->get_request_metadata_cancel_closure, @@ -265,7 +263,6 @@ static void cancel_check_call_host(void* arg, grpc_error* error) { chand->security_connector, &calld->async_result_closure, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_check_call_host"); } static void auth_start_transport_stream_op_batch( @@ -318,7 +315,6 @@ static void auth_start_transport_stream_op_batch( GRPC_ERROR_UNREF(error); } else { // Async return; register cancellation closure with call combiner. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_check_call_host"); grpc_call_combiner_set_notify_on_cancel( calld->call_combiner, GRPC_CLOSURE_INIT(&calld->check_call_host_cancel_closure, diff --git a/src/core/lib/security/transport/server_auth_filter.cc b/src/core/lib/security/transport/server_auth_filter.cc index 2dbefdf1317..19cbb03b63e 100644 --- a/src/core/lib/security/transport/server_auth_filter.cc +++ b/src/core/lib/security/transport/server_auth_filter.cc @@ -156,7 +156,6 @@ static void cancel_call(void* arg, grpc_error* error) { on_md_processing_done_inner(elem, nullptr, 0, nullptr, 0, GRPC_ERROR_REF(error)); } - GRPC_CALL_STACK_UNREF(calld->owning_call, "cancel_call"); } static void recv_initial_metadata_ready(void* arg, grpc_error* error) { @@ -168,7 +167,6 @@ static void recv_initial_metadata_ready(void* arg, grpc_error* error) { if (chand->creds != nullptr && chand->creds->processor.process != nullptr) { // We're calling out to the application, so we need to make sure // to drop the call combiner early if we get cancelled. - GRPC_CALL_STACK_REF(calld->owning_call, "cancel_call"); GRPC_CLOSURE_INIT(&calld->cancel_closure, cancel_call, elem, grpc_schedule_on_exec_ctx); grpc_call_combiner_set_notify_on_cancel(calld->call_combiner, diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index 859915affbc..d81e33054a2 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -613,8 +613,11 @@ void grpc_call_unref(grpc_call* c) { // Unset the call combiner cancellation closure. This has the // effect of scheduling the previously set cancellation closure, if // any, so that it can release any internal references it may be - // holding to the call stack. + // holding to the call stack. Also flush the closures on exec_ctx so that + // filters that schedule cancel notification closures on exec_ctx do not + // need to take a ref of the call stack to guarantee closure liveness. grpc_call_combiner_set_notify_on_cancel(&c->call_combiner, nullptr); + grpc_core::ExecCtx::Get()->Flush(); } GRPC_CALL_INTERNAL_UNREF(c, "destroy"); } diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 0b3508e27ca..10d90e1406c 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -281,6 +281,17 @@ void grpc_channel_get_info(grpc_channel* channel, elem->filter->get_channel_info(elem, channel_info); } +void grpc_channel_reset_connect_backoff(grpc_channel* channel) { + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1, + (channel)); + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->reset_connect_backoff = true; + grpc_channel_element* elem = + grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0); + elem->filter->start_transport_op(elem, op); +} + static grpc_call* grpc_channel_create_call_internal( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, grpc_completion_queue* cq, grpc_pollset_set* pollset_set_alternative, diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h index f01852473ba..6543796b4c8 100644 --- a/src/core/lib/surface/channel_init.h +++ b/src/core/lib/surface/channel_init.h @@ -21,11 +21,37 @@ #include +#include + #include "src/core/lib/channel/channel_stack_builder.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" -#define GRPC_CHANNEL_INIT_BUILTIN_PRIORITY 10000 +// Priority for channel registration functions to be used in +// grpc_channel_init_register_stage(). The priority dictates the +// order in which the registration functions run. +// +// When used to register a filter, the filter can either be appended or +// prepended, thus dictating whether the filter goes at the top or bottom of +// the stack. Higher priority functions can get closer to the top or bottom +// of the stack than lower priority functions. +enum { + // Default level. Most of filters should use this level if their location in + // the stack does not matter. + GRPC_CHANNEL_INIT_PRIORITY_LOW = 0, + // For filters that should be added after the group of filters with default + // priority, such as auth filters. + GRPC_CHANNEL_INIT_PRIORITY_MED = 10000, + // For filters that need to be close to top or bottom, such as protocol-level + // filters (client_authority, http-client, http-server). + GRPC_CHANNEL_INIT_PRIORITY_HIGH = 20000, + // For filters that need to be very close to the wire or surface, such as + // stats filters (census). + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH = 30000, + // For things that have to happen last, such as connected channel filter or + // surface server filter. Consider as reserved for gRPC internals. + GRPC_CHANNEL_INIT_PRIORITY_MAX = INT_MAX +}; /// This module provides a way for plugins (and the grpc core library itself) /// to register mutators for channel stacks. diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index 0ad82fed99e..7807b261d40 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -70,11 +70,6 @@ static void do_basic_init(void) { g_initializations = 0; } -static bool append_filter(grpc_channel_stack_builder* builder, void* arg) { - return grpc_channel_stack_builder_append_filter( - builder, static_cast(arg), nullptr, nullptr); -} - static bool prepend_filter(grpc_channel_stack_builder* builder, void* arg) { return grpc_channel_stack_builder_prepend_filter( builder, static_cast(arg), nullptr, nullptr); @@ -82,19 +77,20 @@ static bool prepend_filter(grpc_channel_stack_builder* builder, void* arg) { static void register_builtin_channel_init() { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_LAME_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - append_filter, (void*)&grpc_lame_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, prepend_filter, - (void*)&grpc_server_top_filter); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + prepend_filter, (void*)&grpc_lame_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, prepend_filter, + (void*)&grpc_server_top_filter); } typedef struct grpc_plugin { diff --git a/src/core/lib/surface/init_secure.cc b/src/core/lib/surface/init_secure.cc index 28c6f7b121f..8058aaa804d 100644 --- a/src/core/lib/surface/init_secure.cc +++ b/src/core/lib/surface/init_secure.cc @@ -67,14 +67,17 @@ static bool maybe_prepend_server_auth_filter( } void grpc_register_security_filters(void) { - // Register the auth client with a priority < INT_MAX to allow the authority + // Register the auth client with a medium priority to allow the authority // filter -on which the auth filter depends- to be higher on the channel // stack. - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX - 1, + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_client_auth_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX - 1, + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_client_auth_filter, nullptr); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_server_auth_filter, nullptr); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 585b9dfae91..9e784635c69 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -282,6 +282,8 @@ typedef struct grpc_transport_op { /** Called when the ping ack is received */ grpc_closure* on_ack; } send_ping; + // If true, will reset the channel's connection backoff. + bool reset_connect_backoff; /*************************************************************************** * remaining fields are initialized and used at the discretion of the diff --git a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc index 06b999899ac..1df1021bb17 100644 --- a/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc +++ b/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc @@ -462,6 +462,14 @@ void alts_tsi_handshaker_handle_response(alts_tsi_handshaker* handshaker, set_unused_bytes(result, &handshaker->recv_bytes, resp->bytes_consumed); } grpc_status_code code = static_cast(resp->status.code); + if (code != GRPC_STATUS_OK) { + grpc_slice* details = static_cast(resp->status.details.arg); + if (details != nullptr) { + char* error_details = grpc_slice_to_c_string(*details); + gpr_log(GPR_ERROR, "Error from handshaker service:%s", error_details); + gpr_free(error_details); + } + } grpc_gcp_handshaker_resp_destroy(resp); cb(alts_tsi_utils_convert_to_tsi_result(code), user_data, bytes_to_send, bytes_to_send_size, result); diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc index 867f31f0255..39b891c2e1f 100644 --- a/src/cpp/client/channel_cc.cc +++ b/src/cpp/client/channel_cc.cc @@ -84,6 +84,14 @@ grpc::string Channel::GetServiceConfigJSON() const { &channel_info.service_config_json); } +namespace experimental { + +void ChannelResetConnectionBackoff(Channel* channel) { + grpc_channel_reset_connect_backoff(channel->c_channel_); +} + +} // namespace experimental + internal::Call Channel::CreateCall(const internal::RpcMethod& method, ClientContext* context, CompletionQueue* cq) { diff --git a/src/cpp/common/channel_filter.cc b/src/cpp/common/channel_filter.cc index 422e7bb65ee..0634b0416fd 100644 --- a/src/cpp/common/channel_filter.cc +++ b/src/cpp/common/channel_filter.cc @@ -78,8 +78,13 @@ bool MaybeAddFilter(grpc_channel_stack_builder* builder, void* arg) { grpc_channel_stack_builder_get_channel_arguments(builder); if (!filter.include_filter(*args)) return true; } - return grpc_channel_stack_builder_prepend_filter(builder, &filter.filter, - nullptr, nullptr); + if (filter.prepend) { + return grpc_channel_stack_builder_prepend_filter(builder, &filter.filter, + nullptr, nullptr); + } else { + return grpc_channel_stack_builder_append_filter(builder, &filter.filter, + nullptr, nullptr); + } } } // namespace diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index 5e569c97e68..359c72737c2 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -36,7 +36,8 @@ /// \c ChannelData. Then register the filter using something like this: /// \code{.cpp} /// RegisterChannelFilter( -/// "name-of-filter", GRPC_SERVER_CHANNEL, INT_MAX, nullptr); +/// "name-of-filter", GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, +/// true, nullptr); /// \endcode namespace grpc { @@ -351,6 +352,7 @@ class ChannelFilter final { struct FilterRecord { grpc_channel_stack_type stack_type; int priority; + bool prepend; std::function include_filter; grpc_channel_filter filter; }; @@ -363,12 +365,14 @@ void ChannelFilterPluginShutdown(); /// Registers a new filter. /// Must be called by only one thread at a time. +/// The \a prepend argument decides whether to prepend or append the filter. /// The \a include_filter argument specifies a function that will be called /// to determine at run-time whether or not to add the filter. If the /// value is nullptr, the filter will be added unconditionally. template void RegisterChannelFilter( const char* name, grpc_channel_stack_type stack_type, int priority, + bool prepend, std::function include_filter) { // If we haven't been called before, initialize channel_filters and // call grpc_register_plugin(). @@ -383,6 +387,7 @@ void RegisterChannelFilter( internal::FilterRecord filter_record = { stack_type, priority, + prepend, include_filter, {FilterType::StartTransportStreamOpBatch, FilterType::StartTransportOp, FilterType::call_data_size, FilterType::InitCallElement, diff --git a/src/cpp/ext/filters/census/grpc_plugin.cc b/src/cpp/ext/filters/census/grpc_plugin.cc index f978ed3bf51..f79e0e0e960 100644 --- a/src/cpp/ext/filters/census/grpc_plugin.cc +++ b/src/cpp/ext/filters/census/grpc_plugin.cc @@ -32,10 +32,12 @@ namespace grpc { void RegisterOpenCensusPlugin() { RegisterChannelFilter( - "opencensus_client", GRPC_CLIENT_CHANNEL, INT_MAX /* priority */, + "opencensus_client", GRPC_CLIENT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, true /* prepend */, nullptr /* condition function */); RegisterChannelFilter( - "opencensus_server", GRPC_SERVER_CHANNEL, INT_MAX /* priority */, + "opencensus_server", GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, true /* prepend */, nullptr /* condition function */); // Access measures to ensure they are initialized. Otherwise, creating a view diff --git a/src/cpp/server/load_reporter/util.cc b/src/cpp/server/load_reporter/util.cc index a2f2f11e707..89bdf57049c 100644 --- a/src/cpp/server/load_reporter/util.cc +++ b/src/cpp/server/load_reporter/util.cc @@ -20,6 +20,8 @@ #include +#include + #include namespace grpc { diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj index fc32271063f..dc5683c9753 100755 --- a/src/csharp/Grpc.Core/Grpc.Core.csproj +++ b/src/csharp/Grpc.Core/Grpc.Core.csproj @@ -47,35 +47,35 @@ true - runtimes/monoandroid/armeabi-v7a/libgrpc_csharp_ext.so + native/android/armeabi-v7a/libgrpc_csharp_ext.so true - runtimes/monoandroid/arm64-v8a/libgrpc_csharp_ext.so + native/android/arm64-v8a/libgrpc_csharp_ext.so true - runtimes/monoandroid/x86/libgrpc_csharp_ext.so + native/android/x86/libgrpc_csharp_ext.so true - runtimes/ios/native/libgrpc_csharp_ext.a + native/ios/universal/libgrpc_csharp_ext.a true - runtimes/ios/native/libgrpc.a + native/ios/universal/libgrpc.a true build/net45/ true - - build/MonoAndroid/ + + build/MonoAndroid10/ true - - build/Xamarin.iOS/ + + build/Xamarin.iOS10/ true diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs index a6a1d8af50e..6ca694e0e46 100644 --- a/src/csharp/Grpc.Core/GrpcEnvironment.cs +++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs @@ -50,6 +50,7 @@ namespace Grpc.Core static int requestCallContextPoolThreadLocalCapacity = DefaultRequestCallContextPoolThreadLocalCapacity; static readonly HashSet registeredChannels = new HashSet(); static readonly HashSet registeredServers = new HashSet(); + static readonly AtomicCounter nativeInitCounter = new AtomicCounter(); static ILogger logger = new LogLevelFilterLogger(new ConsoleLogger(), LogLevel.Off, true); @@ -360,12 +361,25 @@ namespace Grpc.Core internal static void GrpcNativeInit() { + if (!IsNativeShutdownAllowed && nativeInitCounter.Count > 0) + { + // Normally grpc_init and grpc_shutdown calls should come in pairs (C core does reference counting), + // but in case we avoid grpc_shutdown calls altogether, calling grpc_init has no effect + // besides incrementing an internal C core counter that could theoretically overflow. + // To avoid this theoretical possibility we guard repeated calls to grpc_init() + // with a 64-bit atomic counter (that can't realistically overflow). + return; + } NativeMethods.Get().grpcsharp_init(); + nativeInitCounter.Increment(); } internal static void GrpcNativeShutdown() { - NativeMethods.Get().grpcsharp_shutdown(); + if (IsNativeShutdownAllowed) + { + NativeMethods.Get().grpcsharp_shutdown(); + } } /// @@ -411,6 +425,14 @@ namespace Grpc.Core return GetThreadPoolSizeOrDefault(); } + // On some platforms (specifically iOS), thread local variables in native code + // require initialization/destruction. By skipping the grpc_shutdown() call, + // we avoid a potential crash where grpc_shutdown() has already destroyed + // the thread local variables, but some C core's *_destroy() methods still + // need to run (e.g. they may be run by finalizer thread which is out of our control) + // For more context, see https://github.com/grpc/grpc/issues/16294 + private static bool IsNativeShutdownAllowed => !PlatformApis.IsXamarinIOS && !PlatformApis.IsUnityIOS; + private static class ShutdownHooks { static object staticLock = new object(); diff --git a/src/csharp/Grpc.Core/Internal/PlatformApis.cs b/src/csharp/Grpc.Core/Internal/PlatformApis.cs index c501aa89fb6..a8f147545b4 100644 --- a/src/csharp/Grpc.Core/Internal/PlatformApis.cs +++ b/src/csharp/Grpc.Core/Internal/PlatformApis.cs @@ -42,6 +42,7 @@ namespace Grpc.Core.Internal static readonly bool isMono; static readonly bool isNetCore; static readonly bool isUnity; + static readonly bool isUnityIOS; static readonly bool isXamarin; static readonly bool isXamarinIOS; static readonly bool isXamarinAndroid; @@ -63,7 +64,25 @@ namespace Grpc.Core.Internal isNetCore = false; #endif isMono = Type.GetType("Mono.Runtime") != null; - isUnity = Type.GetType(UnityEngineApplicationClassName) != null; + + // Unity + var unityApplicationClass = Type.GetType(UnityEngineApplicationClassName); + if (unityApplicationClass != null) + { + isUnity = true; + // Consult value of Application.platform via reflection + // https://docs.unity3d.com/ScriptReference/Application-platform.html + var platformProperty = unityApplicationClass.GetTypeInfo().GetProperty("platform"); + var unityRuntimePlatform = platformProperty?.GetValue(null)?.ToString(); + isUnityIOS = (unityRuntimePlatform == "IPhonePlayer"); + } + else + { + isUnity = false; + isUnityIOS = false; + } + + // Xamarin isXamarinIOS = Type.GetType(XamarinIOSObjectClassName) != null; isXamarinAndroid = Type.GetType(XamarinAndroidObjectClassName) != null; isXamarin = isXamarinIOS || isXamarinAndroid; @@ -97,6 +116,14 @@ namespace Grpc.Core.Internal get { return isUnity; } } + /// + /// true if running on Unity iOS, false otherwise. + /// + public static bool IsUnityIOS + { + get { return isUnityIOS; } + } + /// /// true if running on a Xamarin platform (either Xamarin.Android or Xamarin.iOS), /// false otherwise. diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 1a746acdd4f..45bd8ebd855 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -2,6 +2,6 @@ 1.15.0-dev - 3.5.1 + 3.6.1 diff --git a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets similarity index 58% rename from src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets rename to src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets index d75e5a2f2f9..250d3bd0cd6 100644 --- a/src/csharp/Grpc.Core/build/MonoAndroid/Grpc.Core.targets +++ b/src/csharp/Grpc.Core/build/MonoAndroid10/Grpc.Core.targets @@ -1,25 +1,22 @@ - - <_GrpcCoreNugetNativePath Condition="'$(_GrpcCoreNugetNativePath)' == ''">$(MSBuildThisFileDirectory)..\..\ - - + Always arm64-v8a - + Always armeabi-v7a - + Always x86 diff --git a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets similarity index 60% rename from src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets rename to src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets index 658158f6ea9..dda1cdd1e84 100644 --- a/src/csharp/Grpc.Core/build/Xamarin.iOS/Grpc.Core.targets +++ b/src/csharp/Grpc.Core/build/Xamarin.iOS10/Grpc.Core.targets @@ -2,11 +2,11 @@ - + Static True - + Static True diff --git a/src/csharp/Grpc.IntegrationTesting/Control.cs b/src/csharp/Grpc.IntegrationTesting/Control.cs index f3284a5390f..6e003484513 100644 --- a/src/csharp/Grpc.IntegrationTesting/Control.cs +++ b/src/csharp/Grpc.IntegrationTesting/Control.cs @@ -1039,14 +1039,14 @@ namespace Grpc.Testing { public ClientConfig(ClientConfig other) : this() { serverTargets_ = other.serverTargets_.Clone(); clientType_ = other.clientType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; outstandingRpcsPerChannel_ = other.outstandingRpcsPerChannel_; clientChannels_ = other.clientChannels_; asyncClientThreads_ = other.asyncClientThreads_; rpcType_ = other.rpcType_; - LoadParams = other.loadParams_ != null ? other.LoadParams.Clone() : null; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; - HistogramParams = other.histogramParams_ != null ? other.HistogramParams.Clone() : null; + loadParams_ = other.loadParams_ != null ? other.loadParams_.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; + histogramParams_ = other.histogramParams_ != null ? other.histogramParams_.Clone() : null; coreList_ = other.coreList_.Clone(); coreLimit_ = other.coreLimit_; otherClientApi_ = other.otherClientApi_; @@ -1647,7 +1647,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStatus(ClientStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -2122,11 +2122,11 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerConfig(ServerConfig other) : this() { serverType_ = other.serverType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; port_ = other.port_; asyncServerThreads_ = other.asyncServerThreads_; coreLimit_ = other.coreLimit_; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; coreList_ = other.coreList_.Clone(); otherServerApi_ = other.otherServerApi_; threadsPerCq_ = other.threadsPerCq_; @@ -2758,7 +2758,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerStatus(ServerStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; port_ = other.port_; cores_ = other.cores_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -3293,9 +3293,9 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public Scenario(Scenario other) : this() { name_ = other.name_; - ClientConfig = other.clientConfig_ != null ? other.ClientConfig.Clone() : null; + clientConfig_ = other.clientConfig_ != null ? other.clientConfig_.Clone() : null; numClients_ = other.numClients_; - ServerConfig = other.serverConfig_ != null ? other.ServerConfig.Clone() : null; + serverConfig_ = other.serverConfig_ != null ? other.serverConfig_.Clone() : null; numServers_ = other.numServers_; warmupSeconds_ = other.warmupSeconds_; benchmarkSeconds_ = other.benchmarkSeconds_; @@ -4422,12 +4422,12 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ScenarioResult(ScenarioResult other) : this() { - Scenario = other.scenario_ != null ? other.Scenario.Clone() : null; - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + scenario_ = other.scenario_ != null ? other.scenario_.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; clientStats_ = other.clientStats_.Clone(); serverStats_ = other.serverStats_.Clone(); serverCores_ = other.serverCores_.Clone(); - Summary = other.summary_ != null ? other.Summary.Clone() : null; + summary_ = other.summary_ != null ? other.summary_.Clone() : null; clientSuccess_ = other.clientSuccess_.Clone(); serverSuccess_ = other.serverSuccess_.Clone(); requestResults_ = other.requestResults_.Clone(); diff --git a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs index 39c3d76ce82..80a1007e9a5 100644 --- a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs +++ b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs @@ -28,7 +28,7 @@ namespace Grpc.Testing { "DGdycGMudGVzdGluZyIyCglEZWJ1Z0luZm8SFQoNc3RhY2tfZW50cmllcxgB", "IAMoCRIOCgZkZXRhaWwYAiABKAkiUAoLRXJyb3JTdGF0dXMSDAoEY29kZRgB", "IAEoBRIVCg1lcnJvcl9tZXNzYWdlGAIgASgJEhwKFGJpbmFyeV9lcnJvcl9k", - "ZXRhaWxzGAMgASgJIuIDCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", + "ZXRhaWxzGAMgASgJIv8DCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", "bmUYASABKAgSHgoWY2xpZW50X2NhbmNlbF9hZnRlcl91cxgCIAEoBRIeChZz", "ZXJ2ZXJfY2FuY2VsX2FmdGVyX3VzGAMgASgFEhUKDWVjaG9fbWV0YWRhdGEY", "BCABKAgSGgoSY2hlY2tfYXV0aF9jb250ZXh0GAUgASgIEh8KF3Jlc3BvbnNl", @@ -39,18 +39,18 @@ namespace Grpc.Testing { "Zy5EZWJ1Z0luZm8SEgoKc2VydmVyX2RpZRgMIAEoCBIcChRiaW5hcnlfZXJy", "b3JfZGV0YWlscxgNIAEoCRIxCg5leHBlY3RlZF9lcnJvchgOIAEoCzIZLmdy", "cGMudGVzdGluZy5FcnJvclN0YXR1cxIXCg9zZXJ2ZXJfc2xlZXBfdXMYDyAB", - "KAUiSgoLRWNob1JlcXVlc3QSDwoHbWVzc2FnZRgBIAEoCRIqCgVwYXJhbRgC", - "IAEoCzIbLmdycGMudGVzdGluZy5SZXF1ZXN0UGFyYW1zIkYKDlJlc3BvbnNl", - "UGFyYW1zEhgKEHJlcXVlc3RfZGVhZGxpbmUYASABKAMSDAoEaG9zdBgCIAEo", - "CRIMCgRwZWVyGAMgASgJIkwKDEVjaG9SZXNwb25zZRIPCgdtZXNzYWdlGAEg", - "ASgJEisKBXBhcmFtGAIgASgLMhwuZ3JwYy50ZXN0aW5nLlJlc3BvbnNlUGFy", - "YW1zYgZwcm90bzM=")); + "KAUSGwoTYmFja2VuZF9jaGFubmVsX2lkeBgQIAEoBSJKCgtFY2hvUmVxdWVz", + "dBIPCgdtZXNzYWdlGAEgASgJEioKBXBhcmFtGAIgASgLMhsuZ3JwYy50ZXN0", + "aW5nLlJlcXVlc3RQYXJhbXMiRgoOUmVzcG9uc2VQYXJhbXMSGAoQcmVxdWVz", + "dF9kZWFkbGluZRgBIAEoAxIMCgRob3N0GAIgASgJEgwKBHBlZXIYAyABKAki", + "TAoMRWNob1Jlc3BvbnNlEg8KB21lc3NhZ2UYASABKAkSKwoFcGFyYW0YAiAB", + "KAsyHC5ncnBjLnRlc3RpbmcuUmVzcG9uc2VQYXJhbXNiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.DebugInfo), global::Grpc.Testing.DebugInfo.Parser, new[]{ "StackEntries", "Detail" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ErrorStatus), global::Grpc.Testing.ErrorStatus.Parser, new[]{ "Code", "ErrorMessage", "BinaryErrorDetails" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs", "BackendChannelIdx" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoRequest), global::Grpc.Testing.EchoRequest.Parser, new[]{ "Message", "Param" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParams), global::Grpc.Testing.ResponseParams.Parser, new[]{ "RequestDeadline", "Host", "Peer" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoResponse), global::Grpc.Testing.EchoResponse.Parser, new[]{ "Message", "Param" }, null, null, null) @@ -435,11 +435,12 @@ namespace Grpc.Testing { expectedClientIdentity_ = other.expectedClientIdentity_; skipCancelledCheck_ = other.skipCancelledCheck_; expectedTransportSecurityType_ = other.expectedTransportSecurityType_; - DebugInfo = other.debugInfo_ != null ? other.DebugInfo.Clone() : null; + debugInfo_ = other.debugInfo_ != null ? other.debugInfo_.Clone() : null; serverDie_ = other.serverDie_; binaryErrorDetails_ = other.binaryErrorDetails_; - ExpectedError = other.expectedError_ != null ? other.ExpectedError.Clone() : null; + expectedError_ = other.expectedError_ != null ? other.expectedError_.Clone() : null; serverSleepUs_ = other.serverSleepUs_; + backendChannelIdx_ = other.backendChannelIdx_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -622,6 +623,20 @@ namespace Grpc.Testing { } } + /// Field number for the "backend_channel_idx" field. + public const int BackendChannelIdxFieldNumber = 16; + private int backendChannelIdx_; + /// + /// which backend to send request to + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int BackendChannelIdx { + get { return backendChannelIdx_; } + set { + backendChannelIdx_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public override bool Equals(object other) { return Equals(other as RequestParams); @@ -650,6 +665,7 @@ namespace Grpc.Testing { if (BinaryErrorDetails != other.BinaryErrorDetails) return false; if (!object.Equals(ExpectedError, other.ExpectedError)) return false; if (ServerSleepUs != other.ServerSleepUs) return false; + if (BackendChannelIdx != other.BackendChannelIdx) return false; return Equals(_unknownFields, other._unknownFields); } @@ -671,6 +687,7 @@ namespace Grpc.Testing { if (BinaryErrorDetails.Length != 0) hash ^= BinaryErrorDetails.GetHashCode(); if (expectedError_ != null) hash ^= ExpectedError.GetHashCode(); if (ServerSleepUs != 0) hash ^= ServerSleepUs.GetHashCode(); + if (BackendChannelIdx != 0) hash ^= BackendChannelIdx.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -744,6 +761,10 @@ namespace Grpc.Testing { output.WriteRawTag(120); output.WriteInt32(ServerSleepUs); } + if (BackendChannelIdx != 0) { + output.WriteRawTag(128, 1); + output.WriteInt32(BackendChannelIdx); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -797,6 +818,9 @@ namespace Grpc.Testing { if (ServerSleepUs != 0) { size += 1 + pb::CodedOutputStream.ComputeInt32Size(ServerSleepUs); } + if (BackendChannelIdx != 0) { + size += 2 + pb::CodedOutputStream.ComputeInt32Size(BackendChannelIdx); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -859,6 +883,9 @@ namespace Grpc.Testing { if (other.ServerSleepUs != 0) { ServerSleepUs = other.ServerSleepUs; } + if (other.BackendChannelIdx != 0) { + BackendChannelIdx = other.BackendChannelIdx; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -936,6 +963,10 @@ namespace Grpc.Testing { ServerSleepUs = input.ReadInt32(); break; } + case 128: { + BackendChannelIdx = input.ReadInt32(); + break; + } } } } @@ -968,7 +999,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoRequest(EchoRequest other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1316,7 +1347,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoResponse(EchoResponse other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Messages.cs b/src/csharp/Grpc.IntegrationTesting/Messages.cs index b5c93babd20..35546f1b671 100644 --- a/src/csharp/Grpc.IntegrationTesting/Messages.cs +++ b/src/csharp/Grpc.IntegrationTesting/Messages.cs @@ -77,7 +77,6 @@ namespace Grpc.Testing { } #region Enums /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of payload that should be returned. /// public enum PayloadType { @@ -269,7 +268,6 @@ namespace Grpc.Testing { public const int TypeFieldNumber = 1; private global::Grpc.Testing.PayloadType type_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of data in body. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -585,12 +583,12 @@ namespace Grpc.Testing { public SimpleRequest(SimpleRequest other) : this() { responseType_ = other.responseType_; responseSize_ = other.responseSize_; - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; fillUsername_ = other.fillUsername_; fillOauthScope_ = other.fillOauthScope_; - ResponseCompressed = other.responseCompressed_ != null ? other.ResponseCompressed.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + responseCompressed_ = other.responseCompressed_ != null ? other.responseCompressed_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -603,7 +601,6 @@ namespace Grpc.Testing { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, server randomly chooses one from other formats. /// @@ -964,7 +961,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public SimpleResponse(SimpleResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; username_ = other.username_; oauthScope_ = other.oauthScope_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -1168,8 +1165,8 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public StreamingInputCallRequest(StreamingInputCallRequest other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1486,7 +1483,7 @@ namespace Grpc.Testing { public ResponseParameters(ResponseParameters other) : this() { size_ = other.size_; intervalUs_ = other.intervalUs_; - Compressed = other.compressed_ != null ? other.Compressed.Clone() : null; + compressed_ = other.compressed_ != null ? other.compressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1693,8 +1690,8 @@ namespace Grpc.Testing { public StreamingOutputCallRequest(StreamingOutputCallRequest other) : this() { responseType_ = other.responseType_; responseParameters_ = other.responseParameters_.Clone(); - Payload = other.payload_ != null ? other.Payload.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1707,7 +1704,6 @@ namespace Grpc.Testing { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, the payload from each response in the stream /// might be of different types. This is to simulate a mixed type of payload @@ -1927,7 +1923,7 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public StreamingOutputCallResponse(StreamingOutputCallResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Stats.cs b/src/csharp/Grpc.IntegrationTesting/Stats.cs index 8160646215a..af83eef7ba8 100644 --- a/src/csharp/Grpc.IntegrationTesting/Stats.cs +++ b/src/csharp/Grpc.IntegrationTesting/Stats.cs @@ -86,7 +86,7 @@ namespace Grpc.Testing { totalCpuTime_ = other.totalCpuTime_; idleCpuTime_ = other.idleCpuTime_; cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -993,13 +993,13 @@ namespace Grpc.Testing { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStats(ClientStats other) : this() { - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; timeElapsed_ = other.timeElapsed_; timeUser_ = other.timeUser_; timeSystem_ = other.timeSystem_; requestResults_ = other.requestResults_.Clone(); cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.Reflection/Reflection.cs b/src/csharp/Grpc.Reflection/Reflection.cs index 84b2a0a8421..e319be5bff7 100644 --- a/src/csharp/Grpc.Reflection/Reflection.cs +++ b/src/csharp/Grpc.Reflection/Reflection.cs @@ -610,7 +610,7 @@ namespace Grpc.Reflection.V1Alpha { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerReflectionResponse(ServerReflectionResponse other) : this() { validHost_ = other.validHost_; - OriginalRequest = other.originalRequest_ != null ? other.OriginalRequest.Clone() : null; + originalRequest_ = other.originalRequest_ != null ? other.originalRequest_.Clone() : null; switch (other.MessageResponseCase) { case MessageResponseOneofCase.FileDescriptorResponse: FileDescriptorResponse = other.FileDescriptorResponse.Clone(); diff --git a/src/csharp/build_unitypackage.bat b/src/csharp/build_unitypackage.bat index 0dcc38ee96d..9c53114b849 100644 --- a/src/csharp/build_unitypackage.bat +++ b/src/csharp/build_unitypackage.bat @@ -68,7 +68,9 @@ copy /Y Grpc.Core\bin\Release\net45\System.Interactive.Async.dll unitypackage\un copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error @rem create a zipfile that will act as a Unity package -powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('unitypackage\unitypackage_skeleton', 'grpc_unity_package.zip');" +cd unitypackage\unitypackage_skeleton +zip -r ..\..\grpc_unity_package.zip Plugins +cd ..\.. copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error goto :EOF diff --git a/src/csharp/experimental/README.md b/src/csharp/experimental/README.md index 212791ac381..bd53cbcd357 100644 --- a/src/csharp/experimental/README.md +++ b/src/csharp/experimental/README.md @@ -17,6 +17,19 @@ Xamarin.iOS - supported architectures: arm64 (iPhone 6+) and x86_64 (iPhone simulator) # Unity -gRPC C# currently doesn't support Unity, but some proof-of-concept -work has been done. There is in-progress effort to provide users -with a pre-built gRPC package that can be used in their projects. + +gRPC C# now has experimental support for Unity. Please try using gRPC with +Unity and provide feedback! + +How to test gRPC in a Unity project +1. Create a Unity project that targets .NET 4.x (Edit -> Project Settings -> Editor -> Scripting Runtime Version). gRPC uses APIs that are only available in .NET4.5+ so this is a requirement. +2. Download the latest development build of `grpc_unity_package.VERSION.zip` from + [daily builds](https://packages.grpc.io/) +3. Extract the `.zip` file in the `Assets` directory in your Unity project +4. Unity IDE will pick up all the bundled files and add them to project automatically. + You should be able to use gRPC and Protobuf in your scripts from now on. + +What's currently bundled in the `grpc_unity_package` +- Grpc.Core and its dependencies +- Google.Protobuf +- Precompiled native libraries for Linux, MacOS, Windows, Android and iOS. diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta index ccaf0c20ec2..6f93617280d 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/arm64-v8a/libgrpc_csharp_ext.so.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: e5beceb1c8fb2403ab3dea319dcd9a2e +guid: e1f44cc7ecd4244448817ccae6de42a3 PluginImporter: externalObjects: {} serializedVersion: 2 diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta index 1560e207fbd..82578436b83 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/armeabi-v7a/libgrpc_csharp_ext.so.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: e5beceb1c8fb2403ab3dea319dcd9a2e +guid: 04fe0e4dcf310416b991e57c99e5d55f PluginImporter: externalObjects: {} serializedVersion: 2 diff --git a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta index 4e216b8b425..3ebdd010e88 100644 --- a/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta +++ b/src/csharp/unitypackage/unitypackage_skeleton/Plugins/Grpc.Core/runtimes/android/x86/libgrpc_csharp_ext.so.meta @@ -1,5 +1,5 @@ fileFormatVersion: 2 -guid: e5beceb1c8fb2403ab3dea319dcd9a2e +guid: 245e3d515096b414fbcdd1fd4160161a PluginImporter: externalObjects: {} serializedVersion: 2 diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 6ad9166b32e..5e9a9a45131 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -101,7 +101,7 @@ Pod::Spec.new do |s| s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/src/objective-c/!ProtoCompiler.podspec b/src/objective-c/!ProtoCompiler.podspec index 12598e616a6..b98339941e5 100644 --- a/src/objective-c/!ProtoCompiler.podspec +++ b/src/objective-c/!ProtoCompiler.podspec @@ -36,7 +36,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler' - v = '3.5.0' + v = '3.6.0' s.version = v s.summary = 'The Protobuf Compiler (protoc) generates Objective-C files from .proto files' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index 9783b064402..8ce88c7db2d 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -220,17 +220,16 @@ static NSString *const kBearerPrefix = @"Bearer "; } - (void)cancel { - [self - maybeFinishWithError:[NSError - errorWithDomain:kGRPCErrorDomain - code:GRPCErrorCodeCancelled - userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]]; - if (!self.isWaitingForToken) { [self cancelCall]; } else { self.isWaitingForToken = NO; } + [self + maybeFinishWithError:[NSError + errorWithDomain:kGRPCErrorDomain + code:GRPCErrorCodeCancelled + userInfo:@{NSLocalizedDescriptionKey : @"Canceled by app"}]]; } - (void)maybeFinishWithError:(NSError *)errorOrNil { @@ -292,6 +291,7 @@ static NSString *const kBearerPrefix = @"Bearer "; // don't want to throw, because the app shouldn't crash for a behavior // that's on the hands of any server to have. Instead we finish and ask // the server to cancel. + [strongSelf cancelCall]; [strongSelf maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeResourceExhausted @@ -300,7 +300,6 @@ static NSString *const kBearerPrefix = @"Bearer "; @"Client does not have enough memory to " @"hold the server response." }]]; - [strongSelf cancelCall]; return; } [strongWriteable enqueueValue:data @@ -530,13 +529,17 @@ static NSString *const kBearerPrefix = @"Bearer "; } - (void)connectivityChanged:(NSNotification *)note { - [self maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain + // Cancel underlying call upon this notification + __strong GRPCCall *strongSelf = self; + if (strongSelf) { + [self cancelCall]; + [self + maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain code:GRPCErrorCodeUnavailable userInfo:@{ NSLocalizedDescriptionKey : @"Connectivity lost." }]]; - // Cancel underlying call upon this notification - [self cancelCall]; + } } @end diff --git a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m index f28e494868e..7781d27ca4f 100644 --- a/src/objective-c/GRPCClient/private/GRPCWrappedCall.m +++ b/src/objective-c/GRPCClient/private/GRPCWrappedCall.m @@ -187,6 +187,7 @@ grpc_slice _details; size_t _detailsCapacity; grpc_metadata_array _trailers; + const char *_errorString; } - (instancetype)init { @@ -200,6 +201,7 @@ _op.data.recv_status_on_client.status_details = &_details; grpc_metadata_array_init(&_trailers); _op.data.recv_status_on_client.trailing_metadata = &_trailers; + _op.data.recv_status_on_client.error_string = &_errorString; if (handler) { // Prevent reference cycle with _handler __weak typeof(self) weakSelf = self; @@ -207,8 +209,9 @@ __strong typeof(self) strongSelf = weakSelf; if (strongSelf) { char *details = grpc_slice_to_c_string(strongSelf->_details); - NSError *error = - [NSError grpc_errorFromStatusCode:strongSelf->_statusCode details:details]; + NSError *error = [NSError grpc_errorFromStatusCode:strongSelf->_statusCode + details:details + errorString:strongSelf->_errorString]; NSDictionary *trailers = [NSDictionary grpc_dictionaryFromMetadataArray:strongSelf->_trailers]; handler(error, trailers); @@ -223,6 +226,7 @@ - (void)dealloc { grpc_metadata_array_destroy(&_trailers); grpc_slice_unref(_details); + gpr_free((void *)_errorString); } @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.h b/src/objective-c/GRPCClient/private/NSError+GRPC.h index e96b7297c28..a63e76ee4d0 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.h +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.h @@ -24,5 +24,7 @@ * Returns nil if the status code is OK. Otherwise, a NSError whose code is one of |GRPCErrorCode| * and whose domain is |kGRPCErrorDomain|. */ -+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details; ++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode + details:(char *)details + errorString:(const char *)errorString; @end diff --git a/src/objective-c/GRPCClient/private/NSError+GRPC.m b/src/objective-c/GRPCClient/private/NSError+GRPC.m index c2e65e4d8a2..199b2ebb6c3 100644 --- a/src/objective-c/GRPCClient/private/NSError+GRPC.m +++ b/src/objective-c/GRPCClient/private/NSError+GRPC.m @@ -23,13 +23,19 @@ NSString *const kGRPCErrorDomain = @"io.grpc"; @implementation NSError (GRPC) -+ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode details:(char *)details { ++ (instancetype)grpc_errorFromStatusCode:(grpc_status_code)statusCode + details:(char *)details + errorString:(const char *)errorString { if (statusCode == GRPC_STATUS_OK) { return nil; } NSString *message = [NSString stringWithCString:details encoding:NSUTF8StringEncoding]; + NSString *debugMessage = [NSString stringWithCString:errorString encoding:NSUTF8StringEncoding]; return [NSError errorWithDomain:kGRPCErrorDomain code:statusCode - userInfo:@{NSLocalizedDescriptionKey : message}]; + userInfo:@{ + NSLocalizedDescriptionKey : message, + NSDebugDescriptionErrorKey : debugMessage + }]; } @end diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 2a169800a09..a0de8ba8990 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -591,4 +591,39 @@ static GRPCProtoMethod *kFullDuplexCallMethod; [self testTimeoutBackoffWithTimeout:0.3 Backoff:0.7]; } +- (void)testErrorDebugInformation { + __weak XCTestExpectation *expectation = [self expectationWithDescription:@"RPC unauthorized."]; + + RMTSimpleRequest *request = [RMTSimpleRequest message]; + request.fillUsername = YES; + request.fillOauthScope = YES; + GRXWriter *requestsWriter = [GRXWriter writerWithValue:[request data]]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kRemoteSSLHost + path:kUnaryCallMethod.HTTPPath + requestsWriter:requestsWriter]; + + call.oauth2AccessToken = @"bogusToken"; + + id responsesWriteable = + [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + XCTFail(@"Received unexpected response: %@", value); + } + completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Finished without error!"); + NSDictionary *userInfo = errorOrNil.userInfo; + NSString *debugInformation = userInfo[NSDebugDescriptionErrorKey]; + XCTAssertNotNil(debugInformation); + XCTAssertNotEqual([debugInformation length], 0); + NSString *challengeHeader = call.oauth2ChallengeHeader; + XCTAssertGreaterThan(challengeHeader.length, 0, @"No challenge in response headers %@", + call.responseHeaders); + [expectation fulfill]; + }]; + + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end diff --git a/src/python/grpcio/grpc/BUILD.bazel b/src/python/grpcio/grpc/BUILD.bazel new file mode 100644 index 00000000000..3f214bf3b05 --- /dev/null +++ b/src/python/grpcio/grpc/BUILD.bazel @@ -0,0 +1,82 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") + +package(default_visibility = ["//visibility:public"]) + +py_binary( + name = "grpcio", + srcs = ["__init__.py"], + deps = [ + ":utilities", + ":auth", + ":plugin_wrapping", + ":channel", + ":interceptor", + ":server", + "//src/python/grpcio/grpc/_cython:cygrpc", + "//src/python/grpcio/grpc/beta", + "//src/python/grpcio/grpc/experimental", + "//src/python/grpcio/grpc/framework", + requirement('enum34'), + requirement('six'), + ], + data = [ + "//:grpc", + ], + main = "__init__.py", + imports = ["../",], +) + +py_library( + name = "auth", + srcs = ["_auth.py"], +) + +py_library( + name = "channel", + srcs = ["_channel.py"], + deps = [ + ":common", + ":grpcio_metadata", + ], +) + +py_library( + name = "common", + srcs = ["_common.py"], +) + +py_library( + name = "grpcio_metadata", + srcs = ["_grpcio_metadata.py"], +) + +py_library( + name = "interceptor", + srcs = ["_interceptor.py"], +) + +py_library( + name = "plugin_wrapping", + srcs = ["_plugin_wrapping.py"], + deps = [ + ":common", + ] +) + +py_library( + name = "server", + srcs = ["_server.py"], + deps = [ + ":common", + ":interceptor", + ], +) + +py_library( + name = "utilities", + srcs = ["_utilities.py"], + deps = [ + ":common", + ], +) + diff --git a/src/python/grpcio/grpc/_cython/BUILD.bazel b/src/python/grpcio/grpc/_cython/BUILD.bazel new file mode 100644 index 00000000000..7124e83dee7 --- /dev/null +++ b/src/python/grpcio/grpc/_cython/BUILD.bazel @@ -0,0 +1,46 @@ +package(default_visibility = ["//visibility:public"]) + +load("//bazel:cython_library.bzl", "pyx_library") + +pyx_library( + name = "cygrpc", + srcs = [ + "__init__.py", + "cygrpc.pxd", + "cygrpc.pyx", + "_cygrpc/grpc_string.pyx.pxi", + "_cygrpc/arguments.pyx.pxi", + "_cygrpc/call.pyx.pxi", + "_cygrpc/channel.pyx.pxi", + "_cygrpc/credentials.pyx.pxi", + "_cygrpc/completion_queue.pyx.pxi", + "_cygrpc/event.pyx.pxi", + "_cygrpc/metadata.pyx.pxi", + "_cygrpc/operation.pyx.pxi", + "_cygrpc/records.pyx.pxi", + "_cygrpc/security.pyx.pxi", + "_cygrpc/server.pyx.pxi", + "_cygrpc/tag.pyx.pxi", + "_cygrpc/time.pyx.pxi", + "_cygrpc/grpc_gevent.pyx.pxi", + "_cygrpc/grpc.pxi", + "_cygrpc/arguments.pxd.pxi", + "_cygrpc/call.pxd.pxi", + "_cygrpc/channel.pxd.pxi", + "_cygrpc/credentials.pxd.pxi", + "_cygrpc/completion_queue.pxd.pxi", + "_cygrpc/event.pxd.pxi", + "_cygrpc/metadata.pxd.pxi", + "_cygrpc/operation.pxd.pxi", + "_cygrpc/records.pxd.pxi", + "_cygrpc/security.pxd.pxi", + "_cygrpc/server.pxd.pxi", + "_cygrpc/tag.pxd.pxi", + "_cygrpc/time.pxd.pxi", + "_cygrpc/grpc_gevent.pxd.pxi", + ], + deps = [ + "//:grpc", + ], +) + diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi index 893df8eac67..aa187e88a62 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi +++ b/src/python/grpcio/grpc/_cython/_cygrpc/channel.pyx.pxi @@ -309,13 +309,18 @@ cdef SegregatedCall _segregated_call( _ChannelState state, int flags, method, host, object deadline, object metadata, CallCredentials credentials, operationses_and_user_tags): cdef _CallState call_state = _CallState() - cdef grpc_completion_queue *c_completion_queue = ( - grpc_completion_queue_create_for_next(NULL)) cdef SegregatedCall segregated_call + cdef grpc_completion_queue *c_completion_queue def on_success(started_tags): state.segregated_call_states.add(call_state) + with state.condition: + if state.open: + c_completion_queue = (grpc_completion_queue_create_for_next(NULL)) + else: + raise ValueError('Cannot invoke RPC on closed channel!') + try: _call( state, call_state, c_completion_queue, on_success, flags, method, host, @@ -443,8 +448,11 @@ cdef class Channel: def check_connectivity_state(self, bint try_to_connect): with self._state.condition: - return grpc_channel_check_connectivity_state( - self._state.c_channel, try_to_connect) + if self._state.open: + return grpc_channel_check_connectivity_state( + self._state.c_channel, try_to_connect) + else: + raise ValueError('Cannot invoke RPC on closed channel!') def watch_connectivity_state( self, grpc_connectivity_state last_observed_state, object deadline): diff --git a/src/python/grpcio/grpc/beta/BUILD.bazel b/src/python/grpcio/grpc/beta/BUILD.bazel new file mode 100644 index 00000000000..731be5cb25b --- /dev/null +++ b/src/python/grpcio/grpc/beta/BUILD.bazel @@ -0,0 +1,58 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "beta", + srcs = ["__init__.py",], + deps = [ + ":client_adaptations", + ":metadata", + ":server_adaptations", + ":implementations", + ":interfaces", + ":utilities", + ], +) + +py_library( + name = "client_adaptations", + srcs = ["_client_adaptations.py"], + imports=["../../",] +) + +py_library( + name = "metadata", + srcs = ["_metadata.py"], +) + +py_library( + name = "server_adaptations", + srcs = ["_server_adaptations.py"], + imports=["../../",], +) + +py_library( + name = "implementations", + srcs = ["implementations.py"], + imports=["../../",], +) + +py_library( + name = "interfaces", + srcs = ["interfaces.py"], + deps = [ + requirement("six"), + ], + imports=["../../",], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + ":implementations", + ":interfaces", + "//src/python/grpcio/grpc/framework/foundation", + ], +) + diff --git a/src/python/grpcio/grpc/experimental/BUILD.bazel b/src/python/grpcio/grpc/experimental/BUILD.bazel new file mode 100644 index 00000000000..6598d02747b --- /dev/null +++ b/src/python/grpcio/grpc/experimental/BUILD.bazel @@ -0,0 +1,27 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "experimental", + srcs = ["__init__.py",], + deps = [ + ":gevent", + ":session_cache", + ], +) + +py_library( + name = "gevent", + srcs = ["gevent.py"], + deps = [ + "//src/python/grpcio/grpc/_cython:cygrpc", + ], +) + +py_library( + name = "session_cache", + srcs = ["session_cache.py"], + deps = [ + "//src/python/grpcio/grpc/_cython:cygrpc", + ], +) diff --git a/src/python/grpcio/grpc/framework/BUILD.bazel b/src/python/grpcio/grpc/framework/BUILD.bazel new file mode 100644 index 00000000000..55b4f4d2dfe --- /dev/null +++ b/src/python/grpcio/grpc/framework/BUILD.bazel @@ -0,0 +1,11 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "framework", + srcs = ["__init__.py",], + deps = [ + "//src/python/grpcio/grpc/framework/common", + "//src/python/grpcio/grpc/framework/foundation", + "//src/python/grpcio/grpc/framework/interfaces", + ], +) diff --git a/src/python/grpcio/grpc/framework/common/BUILD.bazel b/src/python/grpcio/grpc/framework/common/BUILD.bazel new file mode 100644 index 00000000000..9d9ef682c90 --- /dev/null +++ b/src/python/grpcio/grpc/framework/common/BUILD.bazel @@ -0,0 +1,27 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "common", + srcs = ["__init__.py",], + deps = [ + ":cardinality", + ":style", + ], +) + +py_library( + name = "cardinality", + srcs = ["cardinality.py"], + deps = [ + requirement("enum34"), + ], +) + +py_library( + name = "style", + srcs = ["style.py"], + deps = [ + requirement("enum34"), + ], +) diff --git a/src/python/grpcio/grpc/framework/foundation/BUILD.bazel b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel new file mode 100644 index 00000000000..1287fdd44ed --- /dev/null +++ b/src/python/grpcio/grpc/framework/foundation/BUILD.bazel @@ -0,0 +1,61 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "foundation", + srcs = ["__init__.py",], + deps = [ + ":abandonment", + ":callable_util", + ":future", + ":logging_pool", + ":stream_util", + ":stream", + ], +) + +py_library( + name = "abandonment", + srcs = ["abandonment.py"], +) + +py_library( + name = "callable_util", + srcs = ["callable_util.py"], + deps = [ + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "future", + srcs = ["future.py"], + deps = [ + requirement("six"), + ], +) + +py_library( + name = "logging_pool", + srcs = ["logging_pool.py"], + deps = [ + requirement("futures"), + ], +) + +py_library( + name = "stream_util", + srcs = ["stream_util.py"], + deps = [ + ":stream", + ], +) + +py_library( + name = "stream", + srcs = ["stream.py"], + deps = [ + requirement("six"), + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel new file mode 100644 index 00000000000..b81e196cc3c --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/BUILD.bazel @@ -0,0 +1,10 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "interfaces", + srcs = ["__init__.py",], + deps = [ + "//src/python/grpcio/grpc/framework/interfaces/base", + "//src/python/grpcio/grpc/framework/interfaces/face", + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel new file mode 100644 index 00000000000..408a66a6310 --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/base/BUILD.bazel @@ -0,0 +1,29 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "base_lib", + srcs = ["__init__.py",], + deps = [ + ":base", + ":utilities", + ], +) + +py_library( + name = "base", + srcs = ["base.py"], + deps = [ + "//src/python/grpcio/grpc/framework/foundation:abandonment", + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + requirement("enum34"), + ], +) diff --git a/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel new file mode 100644 index 00000000000..e683e7cc426 --- /dev/null +++ b/src/python/grpcio/grpc/framework/interfaces/face/BUILD.bazel @@ -0,0 +1,32 @@ +load("@grpc_python_dependencies//:requirements.bzl", "requirement") +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "face", + srcs = ["__init__.py",], + deps = [ + ":face_lib", + ":utilities", + ], +) + +py_library( + name = "face_lib", + srcs = ["face.py"], + deps = [ + "//src/python/grpcio/grpc/framework/foundation", + "//src/python/grpcio/grpc/framework/common", + requirement("enum34"), + requirement("six"), + ], +) + +py_library( + name = "utilities", + srcs = ["utilities.py"], + deps = [ + "//src/python/grpcio/grpc/framework/common", + "//src/python/grpcio/grpc/framework/foundation:stream", + ":face_lib", + ], +) diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index d6efb49750f..a8158311fb0 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -254,6 +254,8 @@ CORE_SOURCE_FILES = [ 'src/core/lib/security/credentials/plugin/plugin_credentials.cc', 'src/core/lib/security/credentials/ssl/ssl_credentials.cc', 'src/core/lib/security/security_connector/alts_security_connector.cc', + 'src/core/lib/security/security_connector/load_system_roots_fallback.cc', + 'src/core/lib/security/security_connector/load_system_roots_linux.cc', 'src/core/lib/security/security_connector/local_security_connector.cc', 'src/core/lib/security/security_connector/security_connector.cc', 'src/core/lib/security/transport/client_auth_filter.cc', diff --git a/src/python/grpcio_health_checking/setup.py b/src/python/grpcio_health_checking/setup.py index 35c09827ba7..db2edae2cea 100644 --- a/src/python/grpcio_health_checking/setup.py +++ b/src/python/grpcio_health_checking/setup.py @@ -57,7 +57,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 589d0ff5565..b4087d87b49 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -58,7 +58,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_testing/setup.py b/src/python/grpcio_testing/setup.py index eb480a5464a..6ceb1fc5c9d 100644 --- a/src/python/grpcio_testing/setup.py +++ b/src/python/grpcio_testing/setup.py @@ -29,7 +29,7 @@ PACKAGE_DIRECTORIES = { } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 1262e485715..a94c0963ec1 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -41,8 +41,8 @@ INSTALL_REQUIRES = ( 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.5.2.post1', 'six>=1.10', - 'google-auth>=1.0.0', 'requests>=2.14.2') + 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', + 'requests>=2.14.2') if not PY3: INSTALL_REQUIRES += ('futures>=2.2.0',) diff --git a/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel new file mode 100644 index 00000000000..d69186e1fde --- /dev/null +++ b/src/python/grpcio_tests/tests/unit/framework/foundation/BUILD.bazel @@ -0,0 +1,17 @@ +package(default_visibility = ["//visibility:public"]) + +py_library( + name = "stream_testing", + srcs = ["stream_testing.py"], +) + +py_test( + name = "logging_pool_test", + srcs = ["_logging_pool_test.py"], + main = "_logging_pool_test.py", + size = "small", + deps = [ + "//src/python/grpcio/grpc:grpcio", + ], +) + diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 5018bbbfa15..0d5dcbc5433 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -65,6 +65,7 @@ grpc_census_call_set_context_type grpc_census_call_set_context_import; grpc_census_call_get_context_type grpc_census_call_get_context_import; grpc_channel_get_target_type grpc_channel_get_target_import; grpc_channel_get_info_type grpc_channel_get_info_import; +grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; grpc_insecure_channel_create_type grpc_insecure_channel_create_import; grpc_lame_client_channel_create_type grpc_lame_client_channel_create_import; grpc_channel_destroy_type grpc_channel_destroy_import; @@ -316,6 +317,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_census_call_get_context_import = (grpc_census_call_get_context_type) GetProcAddress(library, "grpc_census_call_get_context"); grpc_channel_get_target_import = (grpc_channel_get_target_type) GetProcAddress(library, "grpc_channel_get_target"); grpc_channel_get_info_import = (grpc_channel_get_info_type) GetProcAddress(library, "grpc_channel_get_info"); + grpc_channel_reset_connect_backoff_import = (grpc_channel_reset_connect_backoff_type) GetProcAddress(library, "grpc_channel_reset_connect_backoff"); grpc_insecure_channel_create_import = (grpc_insecure_channel_create_type) GetProcAddress(library, "grpc_insecure_channel_create"); grpc_lame_client_channel_create_import = (grpc_lame_client_channel_create_type) GetProcAddress(library, "grpc_lame_client_channel_create"); grpc_channel_destroy_import = (grpc_channel_destroy_type) GetProcAddress(library, "grpc_channel_destroy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index edb7e98605a..e8e1df72ad3 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -170,6 +170,9 @@ extern grpc_channel_get_target_type grpc_channel_get_target_import; typedef void(*grpc_channel_get_info_type)(grpc_channel* channel, const grpc_channel_info* channel_info); extern grpc_channel_get_info_type grpc_channel_get_info_import; #define grpc_channel_get_info grpc_channel_get_info_import +typedef void(*grpc_channel_reset_connect_backoff_type)(grpc_channel* channel); +extern grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; +#define grpc_channel_reset_connect_backoff grpc_channel_reset_connect_backoff_import typedef grpc_channel*(*grpc_insecure_channel_create_type)(const char* target, const grpc_channel_args* args, void* reserved); extern grpc_insecure_channel_create_type grpc_insecure_channel_create_import; #define grpc_insecure_channel_create grpc_insecure_channel_create_import diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index de13d02e2a3..2a514ed7b68 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -149,6 +149,8 @@ add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later diff --git a/templates/src/csharp/Grpc.Core/Version.csproj.include.template b/templates/src/csharp/Grpc.Core/Version.csproj.include.template index 398b198dbd1..0ec0a08c499 100755 --- a/templates/src/csharp/Grpc.Core/Version.csproj.include.template +++ b/templates/src/csharp/Grpc.Core/Version.csproj.include.template @@ -4,6 +4,6 @@ ${settings.csharp_version} - 3.5.1 + 3.6.1 diff --git a/templates/src/csharp/build_unitypackage.bat.template b/templates/src/csharp/build_unitypackage.bat.template index a53e385f9b1..76ec10dbd90 100755 --- a/templates/src/csharp/build_unitypackage.bat.template +++ b/templates/src/csharp/build_unitypackage.bat.template @@ -70,7 +70,9 @@ copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error @rem create a zipfile that will act as a Unity package - powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('unitypackage\unitypackage_skeleton', 'grpc_unity_package.zip');" + cd unitypackage\unitypackage_skeleton + zip -r ..\..\grpc_unity_package.zip Plugins + cd ..\.. copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error goto :EOF diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index 5c1358f7c37..30b6c5684cc 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include index bc981dc83e1..b34845e01a3 100644 --- a/templates/test/cpp/naming/resolver_component_tests_defs.include +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) diff --git a/test/core/channel/minimal_stack_is_minimal_test.cc b/test/core/channel/minimal_stack_is_minimal_test.cc index e5953acedcf..5b651ed39b7 100644 --- a/test/core/channel/minimal_stack_is_minimal_test.cc +++ b/test/core/channel/minimal_stack_is_minimal_test.cc @@ -85,21 +85,21 @@ int main(int argc, char** argv) { // tests with a default stack errors += - CHECK_STACK("unknown", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, "authority", - "message_size", "deadline", "connected", NULL); + CHECK_STACK("unknown", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, "deadline", + "authority", "message_size", "connected", NULL); errors += CHECK_STACK("unknown", nullptr, GRPC_CLIENT_SUBCHANNEL, "authority", "message_size", "connected", NULL); errors += CHECK_STACK("unknown", nullptr, GRPC_SERVER_CHANNEL, "server", - "message_size", "deadline", "connected", NULL); + "deadline", "message_size", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, - "authority", "message_size", "deadline", "http-client", - "message_compress", "connected", NULL); + "deadline", "authority", "message_size", + "message_compress", "http-client", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_CLIENT_SUBCHANNEL, "authority", - "message_size", "http-client", "message_compress", + "message_size", "message_compress", "http-client", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_SERVER_CHANNEL, "server", - "message_size", "deadline", "http-server", - "message_compress", "connected", NULL); + "deadline", "message_size", "message_compress", + "http-server", "connected", NULL); errors += CHECK_STACK(nullptr, nullptr, GRPC_CLIENT_CHANNEL, "client-channel", NULL); diff --git a/test/core/end2end/tests/filter_call_init_fails.cc b/test/core/end2end/tests/filter_call_init_fails.cc index ab96879fe44..07e14214461 100644 --- a/test/core/end2end/tests/filter_call_init_fails.cc +++ b/test/core/end2end/tests/filter_call_init_fails.cc @@ -438,7 +438,6 @@ static bool maybe_add_server_channel_filter(grpc_channel_stack_builder* builder, // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -457,7 +456,6 @@ static bool maybe_add_client_channel_filter(grpc_channel_stack_builder* builder, // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -476,7 +474,6 @@ static bool maybe_add_client_subchannel_filter( // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -487,13 +484,17 @@ static bool maybe_add_client_subchannel_filter( } static void init_plugin(void) { - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_server_channel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_channel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_subchannel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_channel_filter, nullptr); } diff --git a/test/core/end2end/tests/filter_causes_close.cc b/test/core/end2end/tests/filter_causes_close.cc index a7f4268803f..891c1b8c1fb 100644 --- a/test/core/end2end/tests/filter_causes_close.cc +++ b/test/core/end2end/tests/filter_causes_close.cc @@ -261,8 +261,9 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { } static void init_plugin(void) { - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, 0, maybe_add_filter, - nullptr); + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_add_filter, nullptr); } static void destroy_plugin(void) {} diff --git a/test/core/end2end/tests/filter_latency.cc b/test/core/end2end/tests/filter_latency.cc index a89db7b094b..02a4d079277 100644 --- a/test/core/end2end/tests/filter_latency.cc +++ b/test/core/end2end/tests/filter_latency.cc @@ -314,7 +314,6 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -326,15 +325,15 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { static void init_plugin(void) { gpr_mu_init(&g_mu); - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_server_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, + maybe_add_filter, (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_server_filter); } static void destroy_plugin(void) { gpr_mu_destroy(&g_mu); } diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc index ba3cbfa6d11..6ed1de15c62 100644 --- a/test/core/end2end/tests/filter_status_code.cc +++ b/test/core/end2end/tests/filter_status_code.cc @@ -333,7 +333,6 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { // So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -350,15 +349,15 @@ static void init_plugin(void) { g_client_code_recv = false; g_server_code_recv = false; - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_server_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, + maybe_add_filter, (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_server_filter); } static void destroy_plugin(void) { diff --git a/test/core/security/BUILD b/test/core/security/BUILD index 12aa84d93b3..b7de955cdbe 100644 --- a/test/core/security/BUILD +++ b/test/core/security/BUILD @@ -128,6 +128,27 @@ grpc_cc_test( ], ) +grpc_cc_test( + name = "linux_system_roots_test", + srcs = ["linux_system_roots_test.cc"], + data = [ + "//test/core/security/etc:bundle.pem", + "//test/core/security/etc:test_roots/cert1.pem", + "//test/core/security/etc:test_roots/cert2.pem", + "//test/core/security/etc:test_roots/cert3.pem", + ], + language = "C++", + external_deps = [ + "gtest", + ], + deps = [ + "//:gpr", + "//:grpc", + "//test/core/util:gpr_test_util", + "//test/core/util:grpc_test_util", + ], +) + grpc_cc_test( name = "ssl_credentials_test", srcs = ["ssl_credentials_test.cc"], @@ -219,9 +240,9 @@ grpc_cc_test( deps = [ "//:gpr", "//:grpc", - "//:grpc_base_c", + "//:grpc_base_c", "//:grpc_secure", - "//:tsi", + "//:tsi", "//:tsi_interface", "//test/core/util:gpr_test_util", ], diff --git a/test/core/security/etc/BUILD b/test/core/security/etc/BUILD new file mode 100644 index 00000000000..2c6ab64a3b8 --- /dev/null +++ b/test/core/security/etc/BUILD @@ -0,0 +1,22 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +licenses(["notice"]) # Apache v2 + +exports_files([ + "bundle.pem", + "test_roots/cert1.pem", + "test_roots/cert2.pem", + "test_roots/cert3.pem", +]) diff --git a/test/core/security/etc/README b/test/core/security/etc/README new file mode 100644 index 00000000000..6ba43825861 --- /dev/null +++ b/test/core/security/etc/README @@ -0,0 +1,2 @@ +These files are manual copies of a pem cert from the /etc/ssl/certs/ directory. +They serve only as dummy certificate test files. diff --git a/test/core/security/etc/bundle.pem b/test/core/security/etc/bundle.pem new file mode 100644 index 00000000000..07d7672f830 --- /dev/null +++ b/test/core/security/etc/bundle.pem @@ -0,0 +1,63 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert1.pem b/test/core/security/etc/test_roots/cert1.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert1.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert2.pem b/test/core/security/etc/test_roots/cert2.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert2.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/etc/test_roots/cert3.pem b/test/core/security/etc/test_roots/cert3.pem new file mode 100644 index 00000000000..988cc68aac2 --- /dev/null +++ b/test/core/security/etc/test_roots/cert3.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDdzCCAl+gAwIBAgIIXDPLYixfszIwDQYJKoZIhvcNAQELBQAwPDEeMBwGA1UE +AwwVQXRvcyBUcnVzdGVkUm9vdCAyMDExMQ0wCwYDVQQKDARBdG9zMQswCQYDVQQG +EwJERTAeFw0xMTA3MDcxNDU4MzBaFw0zMDEyMzEyMzU5NTlaMDwxHjAcBgNVBAMM +FUF0b3MgVHJ1c3RlZFJvb3QgMjAxMTENMAsGA1UECgwEQXRvczELMAkGA1UEBhMC +REUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCVhTuXbyo7LjvPpvMp +Nb7PGKw+qtn4TaA+Gke5vJrf8v7MPkfoepbCJI419KkM/IL9bcFyYie96mvr54rM +VD6QUM+A1JX76LWC1BTFtqlVJVfbsVD2sGBkWXppzwO3bw2+yj5vdHLqqjAqc2K+ +SZFhyBH+DgMq92og3AIVDV4VavzjgsG1xZ1kCWyjWZgHJ8cblithdHFsQ/H3NYkQ +4J7sVaE3IqKHBAUsR320HLliKWYoyrfhk/WklAOZuXCFteZI6o1Q/NnezG8HDt0L +cp2AMBYHlT8oDv3FdU9T1nSatCQujgKRz3bFmx5VdJx4IbHwLfELn8LVlhgf8FQi +eowHAgMBAAGjfTB7MB0GA1UdDgQWBBSnpQaxLKYJYO7Rl+lwrrw7GWzbITAPBgNV +HRMBAf8EBTADAQH/MB8GA1UdIwQYMBaAFKelBrEspglg7tGX6XCuvDsZbNshMBgG +A1UdIAQRMA8wDQYLKwYBBAGwLQMEAQEwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3 +DQEBCwUAA4IBAQAmdzTblEiGKkGdLD4GkGDEjKwLVLgfuXvTBznk+j57sj1O7Z8j +vZfza1zv7v1Apt+hk6EKhqzvINB5Ab149xnYJDE0BAGmuhWawyfc2E8PzBhj/5kP +DpFrdRbhIfzYJsdHt6bPWHJxfrrhTZVHO8mvbaG0weyJ9rQPOLXiZNwlz6bb65pc +maHFCN795trV1lpFDMS3wrUU77QR/w4VtfX128a961qn8FYiqTxlVMYVqL2Gns2D +lmh6cYGJ4Qvh6hEbaAjMaZ7snkGeRDImeuKHCnE96+RapNLbxc3G3mB/ufNPRJLv +KrcYPqcZ2Qt9sTdBQrC6YB3y/gkRsPCHe6ed +-----END CERTIFICATE----- diff --git a/test/core/security/linux_system_roots_test.cc b/test/core/security/linux_system_roots_test.cc new file mode 100644 index 00000000000..fce9c8dcc5d --- /dev/null +++ b/test/core/security/linux_system_roots_test.cc @@ -0,0 +1,104 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#include +#include + +#ifdef GPR_LINUX +#include +#include +#include +#include +#include +#include + +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/tmpfile.h" +#include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/security/context/security_context.h" +#include "src/core/lib/security/security_connector/load_system_roots.h" +#include "src/core/lib/security/security_connector/load_system_roots_linux.h" +#include "src/core/lib/security/security_connector/security_connector.h" +#include "src/core/lib/slice/slice_string_helpers.h" +#include "src/core/tsi/ssl_transport_security.h" +#include "src/core/tsi/transport_security.h" +#include "test/core/util/test_config.h" + +#include "gtest/gtest.h" + +#ifndef GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR +#define GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR "GRPC_USE_SYSTEM_SSL_ROOTS" +#endif + +namespace grpc { +namespace { + +TEST(AbsoluteFilePathTest, ConcatenatesCorrectly) { + const char* directory = "nonexistent/test/directory"; + const char* filename = "doesnotexist.txt"; + char result_path[MAXPATHLEN]; + grpc_core::GetAbsoluteFilePath(directory, filename, result_path); + EXPECT_STREQ(result_path, "nonexistent/test/directory/doesnotexist.txt"); +} + +TEST(CreateRootCertsBundleTest, ReturnsEmpty) { + // Test that CreateRootCertsBundle returns an empty slice for null or + // nonexistent cert directories. + grpc_slice result_slice = grpc_core::CreateRootCertsBundle(nullptr); + EXPECT_TRUE(GRPC_SLICE_IS_EMPTY(result_slice)); + grpc_slice_unref(result_slice); + result_slice = grpc_core::CreateRootCertsBundle("does/not/exist"); + EXPECT_TRUE(GRPC_SLICE_IS_EMPTY(result_slice)); + grpc_slice_unref(result_slice); +} + +TEST(CreateRootCertsBundleTest, BundlesCorrectly) { + gpr_setenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR, "true"); + // Test that CreateRootCertsBundle returns a correct slice. + grpc_slice roots_bundle = grpc_empty_slice(); + GRPC_LOG_IF_ERROR( + "load_file", + grpc_load_file("test/core/security/etc/bundle.pem", 1, &roots_bundle)); + // result_slice should have the same content as roots_bundle. + grpc_slice result_slice = + grpc_core::CreateRootCertsBundle("test/core/security/etc/test_roots"); + char* result_str = grpc_slice_to_c_string(result_slice); + char* bundle_str = grpc_slice_to_c_string(roots_bundle); + EXPECT_STREQ(result_str, bundle_str); + // Clean up. + unsetenv(GRPC_USE_SYSTEM_SSL_ROOTS_ENV_VAR); + gpr_free(result_str); + gpr_free(bundle_str); + grpc_slice_unref(roots_bundle); + grpc_slice_unref(result_slice); +} + +} // namespace +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} +#else +int main() { + printf("*** WARNING: this test is only supported on Linux systems ***\n"); + return 0; +} +#endif // GPR_LINUX diff --git a/test/core/security/security_connector_test.cc b/test/core/security/security_connector_test.cc index e4c3ace6b4a..82d77eef8bd 100644 --- a/test/core/security/security_connector_test.cc +++ b/test/core/security/security_connector_test.cc @@ -363,7 +363,7 @@ static void test_ipv6_address_san(void) { namespace grpc_core { namespace { -class TestDefafaultSllRootStore : public DefaultSslRootStore { +class TestDefaultSslRootStore : public DefaultSslRootStore { public: static grpc_slice ComputePemRootCertsForTesting() { return ComputePemRootCerts(); @@ -389,7 +389,7 @@ static void test_default_ssl_roots(void) { gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); grpc_set_ssl_roots_override_callback(override_roots_success); grpc_slice roots = - grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); char* roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); @@ -398,7 +398,7 @@ static void test_default_ssl_roots(void) { /* Now let's set the env var: We should get the contents pointed value instead. */ gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_env_var_file_path); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_env_var) == 0); @@ -407,7 +407,7 @@ static void test_default_ssl_roots(void) { /* Now reset the env var. We should fall back to the value overridden using the api. */ gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); roots_contents = grpc_slice_to_c_string(roots); grpc_slice_unref(roots); GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); @@ -416,10 +416,10 @@ static void test_default_ssl_roots(void) { /* Now setup a permanent failure for the overridden roots and we should get an empty slice. */ grpc_set_ssl_roots_override_callback(override_roots_permanent_failure); - roots = grpc_core::TestDefafaultSllRootStore::ComputePemRootCertsForTesting(); + roots = grpc_core::TestDefaultSslRootStore::ComputePemRootCertsForTesting(); GPR_ASSERT(GRPC_SLICE_IS_EMPTY(roots)); const tsi_ssl_root_certs_store* root_store = - grpc_core::TestDefafaultSllRootStore::GetRootStore(); + grpc_core::TestDefaultSslRootStore::GetRootStore(); GPR_ASSERT(root_store == nullptr); /* Cleanup. */ diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index d492685915d..429497df82f 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -104,6 +104,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_census_call_get_context); printf("%lx", (unsigned long) grpc_channel_get_target); printf("%lx", (unsigned long) grpc_channel_get_info); + printf("%lx", (unsigned long) grpc_channel_reset_connect_backoff); printf("%lx", (unsigned long) grpc_insecure_channel_create); printf("%lx", (unsigned long) grpc_lame_client_channel_create); printf("%lx", (unsigned long) grpc_channel_destroy); diff --git a/test/cpp/common/channel_filter_test.cc b/test/cpp/common/channel_filter_test.cc index 7bdd53f9e7a..9b603ca5b48 100644 --- a/test/cpp/common/channel_filter_test.cc +++ b/test/cpp/common/channel_filter_test.cc @@ -50,7 +50,8 @@ class MyCallData : public CallData { // C-core, we don't accidentally break the C++ filter API. TEST(ChannelFilterTest, RegisterChannelFilter) { grpc::RegisterChannelFilter( - "myfilter", GRPC_CLIENT_CHANNEL, INT_MAX, nullptr); + "myfilter", GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, true, + nullptr); } // TODO(roth): When we have time, add tests for all methods of the diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 3d31c9d8100..c9246f08068 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -1709,7 +1709,7 @@ TEST_P(AsyncEnd2endServerTryCancelTest, ServerBidiStreamingTryCancelAfter) { } std::vector CreateTestScenarios(bool test_secure, - int test_big_limit) { + bool test_message_size_limit) { std::vector scenarios; std::vector credentials_types; std::vector messages; @@ -1731,13 +1731,18 @@ std::vector CreateTestScenarios(bool test_secure, GPR_ASSERT(!credentials_types.empty()); messages.push_back("Hello"); - for (int sz = 1; sz <= test_big_limit; sz *= 32) { - grpc::string big_msg; - for (int i = 0; i < sz * 1024; i++) { - char c = 'a' + (i % 26); - big_msg += c; + if (test_message_size_limit) { + for (size_t k = 1; k < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH / 1024; + k *= 32) { + grpc::string big_msg; + for (size_t i = 0; i < k * 1024; ++i) { + char c = 'a' + (i % 26); + big_msg += c; + } + messages.push_back(big_msg); } - messages.push_back(big_msg); + messages.push_back( + grpc::string(GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH - 10, 'a')); } // TODO (sreek) Renable tests with health check service after the issue @@ -1758,10 +1763,10 @@ std::vector CreateTestScenarios(bool test_secure, } INSTANTIATE_TEST_CASE_P(AsyncEnd2end, AsyncEnd2endTest, - ::testing::ValuesIn(CreateTestScenarios(true, 1024))); + ::testing::ValuesIn(CreateTestScenarios(true, true))); INSTANTIATE_TEST_CASE_P(AsyncEnd2endServerTryCancel, AsyncEnd2endServerTryCancelTest, - ::testing::ValuesIn(CreateTestScenarios(false, 0))); + ::testing::ValuesIn(CreateTestScenarios(false, false))); } // namespace } // namespace testing diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index c5a73a24698..7fe0da8aaec 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -408,6 +408,36 @@ TEST_F(ClientLbEnd2endTest, PickFirstBackOffMinReconnect) { gpr_atm_rel_store(&g_connection_delay_ms, 0); } +TEST_F(ClientLbEnd2endTest, PickFirstResetConnectionBackoff) { + ChannelArguments args; + constexpr int kInitialBackOffMs = 1000; + args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); + const std::vector ports = {grpc_pick_unused_port_or_die()}; + auto channel = BuildChannel("pick_first", args); + auto stub = BuildStub(channel); + SetNextResolution(ports); + // The channel won't become connected (there's no server). + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Bring up a server on the chosen port. + StartServers(1, ports); + const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC); + // Wait for connect, but not long enough. This proves that we're + // being throttled by initial backoff. + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Reset connection backoff. + experimental::ChannelResetConnectionBackoff(channel.get()); + // Wait for connect. Should happen ~immediately. + EXPECT_TRUE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC); + const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0)); + gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited_ms); + // We should have waited less than kInitialBackOffMs. + EXPECT_LT(waited_ms, kInitialBackOffMs); +} + TEST_F(ClientLbEnd2endTest, PickFirstUpdates) { // Start servers and send one RPC per server. const int kNumServers = 3; diff --git a/test/cpp/end2end/filter_end2end_test.cc b/test/cpp/end2end/filter_end2end_test.cc index 88f8f380c3a..a8022823b10 100644 --- a/test/cpp/end2end/filter_end2end_test.cc +++ b/test/cpp/end2end/filter_end2end_test.cc @@ -323,7 +323,8 @@ TEST_F(FilterEnd2endTest, SimpleBidiStreaming) { void RegisterFilter() { grpc::RegisterChannelFilter( - "test-filter", GRPC_SERVER_CHANNEL, INT_MAX, nullptr); + "test-filter", GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, true, + nullptr); } } // namespace diff --git a/test/cpp/interop/client_helper.cc b/test/cpp/interop/client_helper.cc index 29b5a1ed6c2..fb7b7bb7d03 100644 --- a/test/cpp/interop/client_helper.cc +++ b/test/cpp/interop/client_helper.cc @@ -88,20 +88,20 @@ std::shared_ptr CreateChannelForTestCase( std::shared_ptr creds; if (test_case == "compute_engine_creds") { - GPR_ASSERT(FLAGS_use_tls); - creds = GoogleComputeEngineCredentials(); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : GoogleComputeEngineCredentials(); } else if (test_case == "jwt_token_creds") { - GPR_ASSERT(FLAGS_use_tls); grpc::string json_key = GetServiceAccountJsonKey(); std::chrono::seconds token_lifetime = std::chrono::hours(1); - creds = - ServiceAccountJWTAccessCredentials(json_key, token_lifetime.count()); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : ServiceAccountJWTAccessCredentials(json_key, + token_lifetime.count()); } else if (test_case == "oauth2_auth_token") { - grpc::string raw_token = GetOauth2AccessToken(); - creds = AccessTokenCredentials(raw_token); - GPR_ASSERT(creds); + creds = FLAGS_custom_credentials_type == "google_default_credentials" + ? nullptr + : AccessTokenCredentials(GetOauth2AccessToken()); } if (FLAGS_custom_credentials_type.empty()) { transport_security security_type = diff --git a/test/cpp/naming/cancel_ares_query_test.cc b/test/cpp/naming/cancel_ares_query_test.cc index 0d59bf6fb60..dec7c171dc0 100644 --- a/test/cpp/naming/cancel_ares_query_test.cc +++ b/test/cpp/naming/cancel_ares_query_test.cc @@ -45,11 +45,14 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -// TODO: pull in different headers when enabling this -// test on windows. Also set BAD_SOCKET_RETURN_VAL -// to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif namespace { @@ -91,7 +94,13 @@ class FakeNonResponsiveDNSServer { abort(); } } - ~FakeNonResponsiveDNSServer() { close(socket_); } + ~FakeNonResponsiveDNSServer() { +#ifdef GPR_WINDOWS + closesocket(socket_); +#else + close(socket_); +#endif + } private: int socket_; @@ -193,6 +202,38 @@ TEST(CancelDuringAresQuery, TestCancelActiveDNSQuery) { TestCancelActiveDNSQuery(&args); } +#ifdef GPR_WINDOWS + +void MaybePollArbitraryPollsetTwice() { + grpc_pollset* pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size()); + gpr_mu* mu; + grpc_pollset_init(pollset, &mu); + grpc_pollset_worker* worker = nullptr; + // Make a zero timeout poll + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + // Make a second zero-timeout poll (in case the first one + // short-circuited by picking up a previous "kick") + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + grpc_pollset_destroy(pollset); + gpr_free(pollset); +} + +#else + +void MaybePollArbitraryPollsetTwice() {} + +#endif + TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { grpc_core::ExecCtx exec_ctx; ArgsStruct args; @@ -209,6 +250,12 @@ TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { // this test. This test only cares about what happens to fd's that c-ares // opens. TestCancelActiveDNSQuery(&args); + // This test relies on the assumption that cancelling a c-ares query + // will flush out all callbacks on the current exec ctx, which is true + // on posix platforms but not on Windows, because fd shutdown on Windows + // requires a trip through the polling loop to schedule the callback. + // So we need to do extra polling work on Windows to free things up. + MaybePollArbitraryPollsetTwice(); EXPECT_EQ(grpc_iomgr_count_objects_for_testing(), 0u); grpc_pollset_set_destroy(fake_other_pollset_set); } diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index 5dad2ea7af6..1c9d0676b89 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -68,7 +68,7 @@ def main(): 'gtest': False, 'run': False, 'src': ['test/cpp/naming/resolver_component_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util' + unsecure_build_config_suffix, 'grpc_test_util' + unsecure_build_config_suffix, @@ -129,7 +129,7 @@ def main(): 'gtest': True, 'run': True, 'src': ['test/cpp/naming/cancel_ares_query_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util', 'grpc_test_util', diff --git a/test/cpp/naming/manual_run_resolver_component_test.py b/test/cpp/naming/manual_run_resolver_component_test.py new file mode 100644 index 00000000000..fb2157741a0 --- /dev/null +++ b/test/cpp/naming/manual_run_resolver_component_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +# The c-ares test suite doesn't get ran regularly on Windows, but +# this script provides a way to run a lot of the tests manually. +_MSBUILD_CONFIG = os.environ['CONFIG'] +os.chdir(os.path.join('..', '..', os.getcwd())) +# This port is arbitrary, but it needs to be available. +_DNS_SERVER_PORT = 15353 + +subprocess.call([ + sys.executable, + 'test\\cpp\\naming\\resolver_component_tests_runner.py', + '--test_bin_path', 'cmake\\build\\%s\\resolver_component_test.exe' % _MSBUILD_CONFIG, + '--dns_server_bin_path', 'test\\cpp\\naming\\utils\\dns_server.py', + '--records_config_path', 'test\\cpp\\naming\\resolver_test_record_groups.yaml', + '--dns_server_port', str(_DNS_SERVER_PORT), + '--dns_resolver_bin_path', 'test\\cpp\\naming\\utils\\dns_resolver.py', + '--tcp_connect_bin_path', 'test\\cpp\\naming\\utils\\tcp_connect.py', +]) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 6ac548120ca..3dc6e7178cc 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -16,6 +16,8 @@ * */ +#include + #include #include #include @@ -55,8 +57,15 @@ // TODO: pull in different headers when enabling this // test on windows. Also set BAD_SOCKET_RETURN_VAL // to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif using grpc::SubProcess; using std::vector; @@ -241,6 +250,62 @@ void CheckLBPolicyResultLocked(grpc_channel_args* channel_args, } } +#ifdef GPR_WINDOWS +void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { + sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(dummy_port); + ((char*)&addr.sin6_addr)[15] = 1; + for (;;) { + if (gpr_event_get(done_ev)) { + return; + } + std::vector sockets; + for (size_t i = 0; i < 50; i++) { + SOCKET s = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, + WSA_FLAG_OVERLAPPED); + ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL) + << "Failed to create TCP ipv6 socket"; + gpr_log(GPR_DEBUG, "Opened socket: %d", s); + char val = 1; + ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) != + SOCKET_ERROR) + << "Failed to set socketopt reuseaddr. WSA error: " + + std::to_string(WSAGetLastError()); + ASSERT_TRUE(grpc_tcp_set_non_block(s) == GRPC_ERROR_NONE) + << "Failed to set socket non-blocking"; + ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) != SOCKET_ERROR) + << "Failed to bind socket " + std::to_string(s) + + " to [::1]:" + std::to_string(dummy_port) + + ". WSA error: " + std::to_string(WSAGetLastError()); + ASSERT_TRUE(listen(s, 1) != SOCKET_ERROR) + << "Failed to listen on socket " + std::to_string(s) + + ". WSA error: " + std::to_string(WSAGetLastError()); + sockets.push_back(s); + } + // Do a non-blocking accept followed by a close on all of those sockets. + // Do this in a separate loop to try to induce a time window to hit races. + for (size_t i = 0; i < sockets.size(); i++) { + gpr_log(GPR_DEBUG, "non-blocking accept then close on %d", sockets[i]); + ASSERT_TRUE(accept(sockets[i], nullptr, nullptr) == INVALID_SOCKET) + << "Accept on dummy socket unexpectedly accepted actual connection."; + ASSERT_TRUE(WSAGetLastError() == WSAEWOULDBLOCK) + << "OpenAndCloseSocketsStressLoop accept on socket " + + std::to_string(sockets[i]) + + " failed in " + "an unexpected way. " + "WSA error: " + + std::to_string(WSAGetLastError()) + + ". Socket use-after-close bugs are likely."; + ASSERT_TRUE(closesocket(sockets[i]) != SOCKET_ERROR) + << "Failed to close socket: " + std::to_string(sockets[i]) + + ". WSA error: " + std::to_string(WSAGetLastError()); + } + } + return; +} +#else void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { // The goal of this loop is to catch socket // "use after close" bugs within the c-ares resolver by acting @@ -311,6 +376,7 @@ void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { } } } +#endif void CheckResolverResultLocked(void* argsp, grpc_error* err) { EXPECT_EQ(err, GRPC_ERROR_NONE); @@ -372,9 +438,9 @@ void RunResolvesRelevantRecordsTest(void (*OnDoneLocked)(void* arg, args.expected_lb_policy = FLAGS_expected_lb_policy; // maybe build the address with an authority char* whole_uri = nullptr; - GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", - FLAGS_local_dns_server_address.c_str(), - FLAGS_target_name.c_str())); + GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); // create resolver and resolve grpc_core::OrphanablePtr resolver = grpc_core::ResolverRegistry::CreateResolver(whole_uri, nullptr, diff --git a/test/cpp/naming/resolver_component_tests_runner.py b/test/cpp/naming/resolver_component_tests_runner.py index 69386ebeb03..1873eec35bd 100755 --- a/test/cpp/naming/resolver_component_tests_runner.py +++ b/test/cpp/naming/resolver_component_tests_runner.py @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) @@ -112,6 +119,18 @@ wait_until_dns_server_is_up(args, dns_server_subprocess_output) num_test_failures = 0 +test_runner_log('Run test with target: %s' % 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') +current_test_subprocess = subprocess.Popen([ + args.test_bin_path, + '--target_name', 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.', + '--expected_addrs', '5.5.5.5:443,False', + '--expected_chosen_service_config', '', + '--expected_lb_policy', '', + '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port]) +current_test_subprocess.communicate() +if current_test_subprocess.returncode != 0: + num_test_failures += 1 + test_runner_log('Run test with target: %s' % 'srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') current_test_subprocess = subprocess.Popen([ args.test_bin_path, diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 6c4f89d09b8..3c51a00c7b1 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -1,5 +1,13 @@ resolver_tests_common_zone_name: resolver-tests-version-4.grpctestingexp. resolver_component_tests: +- expected_addrs: + - {address: '5.5.5.5:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: no-srv-ipv4-single-target + records: + no-srv-ipv4-single-target: + - {TTL: '2100', data: 5.5.5.5, type: A} - expected_addrs: - {address: '1.2.3.4:1234', is_balancer: true} expected_chosen_service_config: null diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index d14dc18f168..acf4ef8ef10 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -20,8 +20,12 @@ #include -DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); +DEFINE_bool( + enable_ssl, false, + "Whether to use ssl/tls. Deprecated. Use --channel_creds_type=ssl."); +DEFINE_bool(use_auth, false, + "Whether to create default google credentials. Deprecated. Use " + "--channel_creds_type=gdc."); DEFINE_string( access_token, "", "The access token that will be sent to the server to authenticate RPCs."); @@ -29,47 +33,109 @@ DEFINE_string( ssl_target, "", "If not empty, treat the server host name as this for ssl/tls certificate " "validation."); +DEFINE_string( + channel_creds_type, "", + "The channel creds type: insecure, ssl, gdc (Google Default Credentials) " + "or alts."); namespace grpc { namespace testing { -std::shared_ptr CliCredentials::GetCredentials() +grpc::string CliCredentials::GetDefaultChannelCredsType() const { + // Compatibility logic for --enable_ssl. + if (FLAGS_enable_ssl) { + fprintf(stderr, + "warning: --enable_ssl is deprecated. Use " + "--channel_creds_type=ssl.\n"); + return "ssl"; + } + // Compatibility logic for --use_auth. + if (FLAGS_access_token.empty() && FLAGS_use_auth) { + fprintf(stderr, + "warning: --use_auth is deprecated. Use " + "--channel_creds_type=gdc.\n"); + return "gdc"; + } + return "insecure"; +} + +std::shared_ptr +CliCredentials::GetChannelCredentials() const { + if (FLAGS_channel_creds_type.compare("insecure") == 0) { + return grpc::InsecureChannelCredentials(); + } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { + return grpc::SslCredentials(grpc::SslCredentialsOptions()); + } else if (FLAGS_channel_creds_type.compare("gdc") == 0) { + return grpc::GoogleDefaultCredentials(); + } else if (FLAGS_channel_creds_type.compare("alts") == 0) { + return grpc::experimental::AltsCredentials( + grpc::experimental::AltsCredentialsOptions()); + } + fprintf(stderr, + "--channel_creds_type=%s invalid; must be insecure, ssl, gdc or " + "alts.\n", + FLAGS_channel_creds_type.c_str()); + return std::shared_ptr(); +} + +std::shared_ptr CliCredentials::GetCallCredentials() const { if (!FLAGS_access_token.empty()) { if (FLAGS_use_auth) { fprintf(stderr, "warning: use_auth is ignored when access_token is provided."); } - - return grpc::CompositeChannelCredentials( - grpc::SslCredentials(grpc::SslCredentialsOptions()), - grpc::AccessTokenCredentials(FLAGS_access_token)); + return grpc::AccessTokenCredentials(FLAGS_access_token); } + return std::shared_ptr(); +} - if (FLAGS_use_auth) { - return grpc::GoogleDefaultCredentials(); +std::shared_ptr CliCredentials::GetCredentials() + const { + if (FLAGS_channel_creds_type.empty()) { + FLAGS_channel_creds_type = GetDefaultChannelCredsType(); + } else if (FLAGS_enable_ssl && FLAGS_channel_creds_type.compare("ssl") != 0) { + fprintf(stderr, + "warning: ignoring --enable_ssl because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); + } else if (FLAGS_use_auth && FLAGS_channel_creds_type.compare("gdc") != 0) { + fprintf(stderr, + "warning: ignoring --use_auth because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); } - - if (FLAGS_enable_ssl) { - return grpc::SslCredentials(grpc::SslCredentialsOptions()); + // Legacy transport upgrade logic for insecure requests. + if (!FLAGS_access_token.empty() && + FLAGS_channel_creds_type.compare("insecure") == 0) { + fprintf(stderr, + "warning: --channel_creds_type=insecure upgraded to ssl because " + "an access token was provided.\n"); + FLAGS_channel_creds_type = "ssl"; } - - return grpc::InsecureChannelCredentials(); + std::shared_ptr channel_creds = + GetChannelCredentials(); + // Composite any call-type credentials on top of the base channel. + std::shared_ptr call_creds = GetCallCredentials(); + return (channel_creds == nullptr || call_creds == nullptr) + ? channel_creds + : grpc::CompositeChannelCredentials(channel_creds, call_creds); } const grpc::string CliCredentials::GetCredentialUsage() const { - return " --enable_ssl ; Set whether to use tls\n" + return " --enable_ssl ; Set whether to use ssl (deprecated)\n" " --use_auth ; Set whether to create default google" " credentials\n" " --access_token ; Set the access token in metadata," " overrides --use_auth\n" - " --ssl_target ; Set server host for tls validation\n"; + " --ssl_target ; Set server host for ssl validation\n" + " --channel_creds_type ; Set to insecure, ssl, gdc, or alts\n"; } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - bool use_tls = - FLAGS_enable_ssl || (FLAGS_access_token.empty() && FLAGS_use_auth); - return use_tls ? FLAGS_ssl_target : ""; + bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || + FLAGS_channel_creds_type.compare("gdc") == 0; + return use_ssl ? FLAGS_ssl_target : ""; } } // namespace testing diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 8d662356de8..4636d3ca149 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -28,9 +28,22 @@ namespace testing { class CliCredentials { public: virtual ~CliCredentials() {} - virtual std::shared_ptr GetCredentials() const; + std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; virtual const grpc::string GetSslTargetNameOverride() const; + + protected: + // Returns the appropriate channel_creds_type value for the set of legacy + // flag arguments. + virtual grpc::string GetDefaultChannelCredsType() const; + // Returns the base transport channel credentials. Child classes can override + // to support additional channel_creds_types unknown to this base class. + virtual std::shared_ptr GetChannelCredentials() + const; + // Returns call credentials to composite onto the base transport channel + // credentials. Child classes can override to support additional + // authentication flags unknown to this base class. + virtual std::shared_ptr GetCallCredentials() const; }; } // namespace testing diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 7e7f44551ef..3aae090e818 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -81,7 +81,7 @@ using grpc::testing::EchoResponse; " peer: \"peer\"\n" \ "}\n\n" -DECLARE_bool(enable_ssl); +DECLARE_string(channel_creds_type); DECLARE_string(ssl_target); namespace grpc { @@ -102,7 +102,8 @@ const int kServerDefaultResponseStreamsToSend = 3; class TestCliCredentials final : public grpc::testing::CliCredentials { public: TestCliCredentials(bool secure = false) : secure_(secure) {} - std::shared_ptr GetCredentials() const override { + std::shared_ptr GetChannelCredentials() + const override { if (!secure_) { return InsecureChannelCredentials(); } @@ -769,12 +770,12 @@ TEST_F(GrpcToolTest, CallCommandWithBadMetadata) { TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { const grpc::string server_address = SetUpServer(true); - // Test input "grpc_cli ls localhost: --enable_ssl + // Test input "grpc_cli ls localhost: --channel_creds_type=ssl // --ssl_target=z.test.google.fr" std::stringstream output_stream; const char* argv[] = {"grpc_cli", "ls", server_address.c_str()}; FLAGS_l = false; - FLAGS_enable_ssl = true; + FLAGS_channel_creds_type = "ssl"; FLAGS_ssl_target = "z.test.google.fr"; EXPECT_TRUE( 0 == GrpcToolMainLib( @@ -784,7 +785,7 @@ TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { "grpc.testing.EchoTestService\n" "grpc.reflection.v1alpha.ServerReflection\n")); - FLAGS_enable_ssl = false; + FLAGS_channel_creds_type = ""; FLAGS_ssl_target = ""; ShutdownServer(); } diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat index 8eb3b201b13..7658156be1d 100644 --- a/test/distrib/cpp/run_distrib_test_cmake.bat +++ b/test/distrib/cpp/run_distrib_test_cmake.bat @@ -39,7 +39,7 @@ cd ../../.. cd third_party/protobuf/cmake mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. +cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DZLIB_ROOT=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. cmake --build . --config Release --target install || goto :error cd ../../../.. diff --git a/test/distrib/cpp/run_distrib_test_routeguide.sh b/test/distrib/cpp/run_distrib_test_routeguide.sh index dc69ab83773..a81692d920c 100755 --- a/test/distrib/cpp/run_distrib_test_routeguide.sh +++ b/test/distrib/cpp/run_distrib_test_routeguide.sh @@ -19,7 +19,7 @@ set -ex cd "$(dirname "$0")/../../.." cd third_party/protobuf && ./autogen.sh && \ -./configure && make -j4 && make check && make install && ldconfig +./configure && make -j4 && make install && ldconfig cd ../.. && make -j4 && make install diff --git a/third_party/protobuf b/third_party/protobuf index b5fbb742af1..48cb18e5c41 160000 --- a/third_party/protobuf +++ b/third_party/protobuf @@ -1 +1 @@ -Subproject commit b5fbb742af122b565925987e65c08957739976a7 +Subproject commit 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index 5c781761ba1..15495917467 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -68,7 +68,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.5.2 +pip install protobuf==3.6.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index a0e1419ac12..7d10db0329c 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -14,10 +14,10 @@ # limitations under the License. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="b5fbb742af122b565925987e65c08957739976a7" +PROTOBUF_SUBMODULE_VERSION="48cb18e5c419ddd23d9badcfe4e9df7bde1979b2" diff --git a/tools/dockerfile/OWNERS b/tools/dockerfile/OWNERS index db4ab546a6d..5f0ad58d617 100644 --- a/tools/dockerfile/OWNERS +++ b/tools/dockerfile/OWNERS @@ -7,5 +7,5 @@ set noparent # for kokoro to be able to access the pre-built images. @jtattermusch -@matt-kwong +@mehrdada @nicolasnoble diff --git a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile index 3e1faafdc04..e32b3cb5e25 100644 --- a/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_centos7_x64/Dockerfile @@ -22,3 +22,6 @@ RUN yum install -y mono-devel RUN yum install -y nuget RUN yum install -y unzip + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile index 03fb7a53438..e95d781dfd1 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile index f2fa61a6916..aec936a5b8c 100644 --- a/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_jessie_x86/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile index 3edc31e1703..61ca1a08a46 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1404_x64/Dockerfile @@ -38,3 +38,7 @@ RUN mkdir warmup \ && dotnet new \ && cd .. \ && rm -rf warmup + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile index 1a58f9784bf..93ee75cfcd4 100644 --- a/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/distribtest/csharp_ubuntu1604_x64/Dockerfile @@ -25,3 +25,7 @@ RUN apt-get update && apt-get install -y \ && apt-get clean RUN apt-get update && apt-get install -y unzip && apt-get clean + +# Make sure the mono certificate store is up-to-date to prevent issues with nuget restore +RUN apt-get update && apt-get install -y curl && apt-get clean +RUN curl https://curl.haxx.se/ca/cacert.pem > ~/cacert.pem && cert-sync ~/cacert.pem && rm -f ~/cacert.pem diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 33e2f2970c3..44fde4f99f0 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -30,24 +30,34 @@ RUN yum install -y git \ glibc-devel \ glibc-devel.i686 -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d -RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' -RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" -RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +# Install GCC 4.8 +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo # We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by # docker issue when using overlay storage driver, but all the stuff we need # will be installed, so for now we just ignore the error. # https://github.com/docker/docker/issues/10180 -RUN yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 || true +RUN yum install -y devtoolset-2-build \ + devtoolset-2-toolchain \ + devtoolset-2-binutils \ + devtoolset-2-gcc \ + devtoolset-2-gcc-c++ \ + devtoolset-2-libstdc++-devel \ + devtoolset-2-libstdc++-devel.i686 || true + +# Again, ignore the "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error. +RUN yum install -y ca-certificates || true # renew certs to prevent download error for ius-release.rpm + +# TODO(jtattermusch): gRPC makefile uses "which" to detect the availability of gcc +RUN yum install -y which || true # Update Git to version >1.7 to allow cloning submodules with --reference arg. RUN yum remove -y git && yum clean all RUN yum install -y https://centos6.iuscommunity.org/ius-release.rpm && yum clean all RUN yum install -y git2u && yum clean all -# Start in devtoolset environment that uses GCC 4.7 -CMD ["scl", "enable", "devtoolset-1.1", "bash"] +# Start in devtoolset environment that uses GCC 4.8 +CMD ["scl", "enable", "devtoolset-2", "bash"] diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index 2f06bda0168..688c271feac 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index a46ebe61976..592b0b20e65 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -791,6 +791,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core b/tools/doxygen/Doxyfile.core index 4899eee3ea4..aa75bc68283 100644 --- a/tools/doxygen/Doxyfile.core +++ b/tools/doxygen/Doxyfile.core @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 18f56984fe2..fa2ad93a456 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -793,6 +793,7 @@ doc/server-reflection.md \ doc/server_reflection_tutorial.md \ doc/server_side_auth.md \ doc/service_config.md \ +doc/ssl-performance.md \ doc/status_ordering.md \ doc/statuscodes.md \ doc/unit_testing.md \ @@ -1355,6 +1356,10 @@ src/core/lib/security/credentials/ssl/ssl_credentials.cc \ src/core/lib/security/credentials/ssl/ssl_credentials.h \ src/core/lib/security/security_connector/alts_security_connector.cc \ src/core/lib/security/security_connector/alts_security_connector.h \ +src/core/lib/security/security_connector/load_system_roots.h \ +src/core/lib/security/security_connector/load_system_roots_fallback.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.cc \ +src/core/lib/security/security_connector/load_system_roots_linux.h \ src/core/lib/security/security_connector/local_security_connector.cc \ src/core/lib/security/security_connector/local_security_connector.h \ src/core/lib/security/security_connector/security_connector.cc \ diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc index b0feeef363c..43bc9609c7e 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc @@ -17,17 +17,6 @@ # builds. This rc script must be used in the root directory of gRPC # and is expected to be used before prepare_build_macos_rc -export CONFIG=opt - -# Move gRPC repo to directory that Docker for Mac has drive access to -mkdir /Users/kbuilder/workspace -cp -R ./ /Users/kbuilder/workspace/grpc -cd /Users/kbuilder/workspace/grpc - -# Needed for identifying Docker image sha1 -brew update -brew install md5sha1sum - # Set up gRPC-Go and gRPC-Java to test git clone --recursive https://github.com/grpc/grpc-go ./../grpc-go git clone --recursive https://github.com/grpc/grpc-java ./../grpc-java diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 0164e4a5617..f987f8a8cb5 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -14,7 +14,7 @@ @rem make sure msys binaries are preferred over cygwin binaries @rem set path to python 2.7 -set PATH=C:\tools\msys64\usr\bin;C:\Python27;C:\Python37;%PATH% +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( @@ -34,9 +34,6 @@ netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3 @rem Needed for big_query_utils python -m pip install google-api-python-client -@rem Install Python 3.7 -chocolatey install -y -r python3 --version 3.7 - @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 2220145dacc..42c7f5fb042 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -25,13 +25,6 @@ git submodule update --init # Build protoc and grpc_cpp_plugin. Codegen is not cross-compiled to Android make HAS_SYSTEM_PROTOBUF=false -# TODO(ericgribkoff) Remove when this commit (already in master) is included in -# next protobuf release -cd third_party/protobuf -git fetch -git cherry-pick 7daa320065f3bea2b54bf983337d1724f153422d -m 1 - - # Build and run interop instrumentation tests on Firebase Test Lab cd "${REPO_ROOT}/src/android/test/interop/" diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 5099fa03016..a6367ad9628 100644 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -14,6 +14,6 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 192d9d1b2d0..51cb66f5b86 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 6fb3c77892e..cbba9067ad3 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index ef943db7e15..14492301cc9 100755 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -65,6 +65,7 @@ UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" CSHARP_PACKAGES=( "$UNZIPPED_CSHARP_PACKAGES"/* + "$INPUT_ARTIFACTS"/grpc_unity_package.[0-9]*.zip ) # Python diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 25531ead2b5..2ba7d469ecf 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index d9b039f1ecd..338b1b6a0d0 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -61,8 +61,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --runs_per_test_detects_flakes \ - --runs_per_test=2 \ + --cache_test_results=no \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh index 5ddabb9bf9c..e748a62e761 100755 --- a/tools/internal_ci/macos/grpc_interop_toprod.sh +++ b/tools/internal_ci/macos/grpc_interop_toprod.sh @@ -18,8 +18,8 @@ set -ex # change to grpc repo root cd $(dirname $0)/../../.. -source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc source tools/internal_ci/helper_scripts/prepare_build_macos_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc # using run_interop_tests.py without --use_docker, so we need to build first tools/run_tests/run_tests.py -l c++ -c opt --build_only diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 4d528e01285..0e02bdb6ecc 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -18,6 +18,7 @@ rename C:\Python27_32bit Python27_32bits rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits +rename C:\Python37_32bit Python37_32bits @rem enter repo root cd /d %~dp0\..\..\.. diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 40c02a1bc7a..db84d9b454c 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -9,7 +9,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are: - Add (or update) an entry in `./client_matrix.py` file to reference the github tag for the release. - Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do - - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language cxx csharp python ruby php` + - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php` - Verify that the new docker image was built successfully and uploaded to GCR. For example, - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index fae78909daf..bb9222d953f 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -96,6 +96,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'go': [ { @@ -178,6 +181,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.1': None }, + { + 'v1.14.0': None + }, ], 'python': [ { @@ -219,6 +225,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'node': [ { @@ -304,6 +313,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'php': [ { @@ -345,6 +357,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], 'csharp': [ { @@ -391,6 +406,9 @@ LANG_RELEASE_MATRIX = { { 'v1.13.0': None }, + { + 'v1.14.1': None + }, ], } diff --git a/tools/profiling/ios_bin/binary_size.py b/tools/profiling/ios_bin/binary_size.py index 91c43f44243..d4d134fef37 100755 --- a/tools/profiling/ios_bin/binary_size.py +++ b/tools/profiling/ios_bin/binary_size.py @@ -75,6 +75,7 @@ def get_size(where, frameworks): def build(where, frameworks): + subprocess.check_call(['make', 'clean']) shutil.rmtree( 'src/objective-c/examples/Sample/Build/Build-%s' % where, ignore_errors=True) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 01323b53266..bdeb258e1fc 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -290,15 +290,9 @@ class PHPArtifact: return [] def build_jobspec(self): - if self.platform == 'linux': - return create_docker_jobspec( - self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( - self.arch), - 'tools/run_tests/artifacts/build_artifact_php.sh') - else: - return create_jobspec( - self.name, ['tools/run_tests/artifacts/build_artifact_php.sh'], - use_workspace=True) + return create_docker_jobspec( + self.name, 'tools/dockerfile/grpc_artifact_linux_{}'.format( + self.arch), 'tools/run_tests/artifacts/build_artifact_php.sh') class ProtocArtifact: @@ -392,6 +386,7 @@ def targets(): PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), + PythonArtifact('windows', 'x86', 'Python37_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), @@ -399,6 +394,5 @@ def targets(): PythonArtifact('windows', 'x64', 'Python37'), RubyArtifact('linux', 'x64'), RubyArtifact('macos', 'x64'), - PHPArtifact('linux', 'x64'), - PHPArtifact('macos', 'x64') + PHPArtifact('linux', 'x64') ]) diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index b531fc9a5d4..6d433f2dad4 100755 --- a/tools/run_tests/artifacts/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use devtoolset environment that has GCC 4.7 before set -ex -source scl_source enable devtoolset-1.1 +# Use devtoolset environment that has GCC 4.8 before set -ex +source scl_source enable devtoolset-2 set -ex diff --git a/tools/run_tests/dockerize/build_and_run_docker.sh b/tools/run_tests/dockerize/build_and_run_docker.sh index 4ef74085f94..3f01fbc7b7d 100755 --- a/tools/run_tests/dockerize/build_and_run_docker.sh +++ b/tools/run_tests/dockerize/build_and_run_docker.sh @@ -73,6 +73,10 @@ docker run \ # Copy output artifacts if [ "$OUTPUT_DIR" != "" ] then + # Create the artifact directory in advance to avoid a race in "docker cp" if tasks + # that were running in parallel finish at the same time. + # see https://github.com/grpc/grpc/issues/16155 + mkdir -p "$git_root/$OUTPUT_DIR" docker cp "$CONTAINER_NAME:/var/local/git/grpc/$OUTPUT_DIR" "$git_root" || FAILED="true" fi diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index a686dae8b4a..34e23f09c2d 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3707,6 +3707,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "src": [ + "test/core/security/linux_system_roots_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "grpc_plugin_support" @@ -10358,6 +10375,8 @@ "src/core/lib/security/credentials/plugin/plugin_credentials.h", "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.h", "src/core/lib/security/transport/auth_filters.h", @@ -10406,6 +10425,10 @@ "src/core/lib/security/credentials/ssl/ssl_credentials.h", "src/core/lib/security/security_connector/alts_security_connector.cc", "src/core/lib/security/security_connector/alts_security_connector.h", + "src/core/lib/security/security_connector/load_system_roots.h", + "src/core/lib/security/security_connector/load_system_roots_fallback.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.cc", + "src/core/lib/security/security_connector/load_system_roots_linux.h", "src/core/lib/security/security_connector/local_security_connector.cc", "src/core/lib/security/security_connector/local_security_connector.h", "src/core/lib/security/security_connector/security_connector.cc", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 5815f82fefb..de64097ec1c 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -4333,6 +4333,30 @@ ], "uses_polling": true }, + { + "args": [], + "benchmark": false, + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_linux_system_roots_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "uses_polling": true + }, { "args": [], "benchmark": false, @@ -5784,7 +5808,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5796,7 +5821,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, diff --git a/tools/run_tests/performance/OWNERS b/tools/run_tests/performance/OWNERS index fc1d6eec9c7..9cf8c131111 100644 --- a/tools/run_tests/performance/OWNERS +++ b/tools/run_tests/performance/OWNERS @@ -5,5 +5,5 @@ set noparent # to update the BigQuery schema @ncteisen -@matt-kwong +@apolcyn @jtattermusch diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index aa58107ced9..22055d58e8b 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -637,13 +637,13 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 'java', 'go', 'python', 'c++' ] -#TODO: Add c++ when c++ ALTS interop client is ready. _LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -#TODO: Add c++ when c++ ALTS interop server is ready. _SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go', 'c++'] -_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] +_TRANSPORT_SECURITY_OPTIONS = [ + 'tls', 'alts', 'google_default_credentials', 'insecure' +] DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -724,6 +724,9 @@ def auth_options(language, test_case, service_account_key_file=None): key_file_arg = '--service_account_key_file=%s' % service_account_key_file default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' + # TODO: When using google_default_credentials outside of cloud-to-prod, the environment variable + # 'GOOGLE_APPLICATION_CREDENTIALS' needs to be set for the test case + # 'jwt_token_creds' to work. if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: if language in [ 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', @@ -763,15 +766,25 @@ def cloud_to_prod_jobspec(language, docker_image=None, auth=False, manual_cmd_log=None, - service_account_key_file=None): + service_account_key_file=None, + transport_security='tls'): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ '--server_host=%s' % server_host, '--server_host_override=%s' % server_host, '--server_port=443', - '--use_tls=true', '--test_case=%s' % test_case ] + if transport_security == 'tls': + transport_security_options += ['--use_tls=true'] + elif transport_security == 'google_default_credentials' and language == 'c++': + transport_security_options += [ + '--custom_credentials_type=google_default_credentials' + ] + else: + print('Invalid transport security option.') + sys.exit(1) + cmdargs = cmdargs + transport_security_options environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: auth_cmdargs, auth_env = auth_options(language, test_case, @@ -1285,14 +1298,16 @@ try: jobs = [] if args.cloud_to_prod: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _TEST_CASES: if not test_case in language.unimplemented_test_cases(): if not test_case in _SKIP_ADVANCED + _SKIP_COMPRESSION: - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1300,8 +1315,23 @@ try: docker_image=docker_images.get(str(language)), manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) if args.http2_interop: for test_case in _HTTP2_TEST_CASES: @@ -1312,12 +1342,15 @@ try: prod_servers[server_host_nickname], docker_image=docker_images.get(str(http2Interop)), manual_cmd_log=client_manual_cmd_log, - service_account_key_file=args.service_account_key_file) + service_account_key_file=args.service_account_key_file, + transport_security=args.transport_security) jobs.append(test_job) if args.cloud_to_prod_auth: - if args.transport_security != 'tls': - print('TLS is always enabled for cloud_to_prod scenarios.') + if args.transport_security not in ['tls', 'google_default_credentials']: + print( + 'TLS or google default credential is always enabled for cloud_to_prod scenarios.' + ) for server_host_nickname in args.prod_servers: for language in languages: for test_case in _AUTH_TEST_CASES: @@ -1325,7 +1358,7 @@ try: not compute_engine_creds_required( language, test_case)): if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec( + tls_test_job = cloud_to_prod_jobspec( language, test_case, server_host_nickname, @@ -1334,8 +1367,23 @@ try: auth=True, manual_cmd_log=client_manual_cmd_log, service_account_key_file=args. - service_account_key_file) - jobs.append(test_job) + service_account_key_file, + transport_security='tls') + jobs.append(tls_test_job) + if language == 'c++': + google_default_creds_test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_nickname, + prod_servers[server_host_nickname], + docker_image=docker_images.get( + str(language)), + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file, + transport_security= + 'google_default_credentials') + jobs.append(google_default_creds_test_job) for server in args.override_server: server_name = server[0] diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index fb159bc2d20..2c7c140716f 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -36,7 +36,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) - b5fbb742af122b565925987e65c08957739976a7 third_party/protobuf (v3.3.1-641-gb5fbb74) + 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 third_party/protobuf (v3.6.0.1-37-g48cb18e5) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF