From b3d70102a59d02e238417f2d8fc4eaea66bad028 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 31 Mar 2017 14:17:09 -0700 Subject: [PATCH 01/60] Memory allocator for C++ in core --- CMakeLists.txt | 39 ++++ Makefile | 48 +++++ build.yaml | 15 ++ gRPC-Core.podspec | 2 + grpc.gemspec | 1 + package.xml | 1 + src/core/lib/support/memory.h | 74 +++++++ test/core/support/memory_test.cc | 76 +++++++ tools/doxygen/Doxyfile.core.internal | 1 + .../generated/sources_and_headers.json | 21 ++ tools/run_tests/generated/tests.json | 22 ++ vsprojects/vcxproj/gpr/gpr.vcxproj | 1 + vsprojects/vcxproj/gpr/gpr.vcxproj.filters | 3 + .../test/memory_test/memory_test.vcxproj | 204 ++++++++++++++++++ .../memory_test/memory_test.vcxproj.filters | 21 ++ 15 files changed, 529 insertions(+) create mode 100644 src/core/lib/support/memory.h create mode 100644 test/core/support/memory_test.cc create mode 100644 vsprojects/vcxproj/test/memory_test/memory_test.vcxproj create mode 100644 vsprojects/vcxproj/test/memory_test/memory_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 41821bec3c1..f75231cfe14 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -685,6 +685,7 @@ endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx json_run_localhost) endif() +add_dependencies(buildtests_cxx memory_test) add_dependencies(buildtests_cxx metrics_client) add_dependencies(buildtests_cxx mock_test) add_dependencies(buildtests_cxx noop-benchmark) @@ -10420,6 +10421,44 @@ endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(memory_test + test/core/support/memory_test.cc + third_party/googletest/src/gtest-all.cc +) + + +target_include_directories(memory_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include + PRIVATE third_party/googletest/include + PRIVATE third_party/googletest + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(memory_test + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc++ + grpc + gpr_test_util + gpr + ${_gRPC_GFLAGS_LIBRARIES} +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(metrics_client ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/metrics.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/metrics.grpc.pb.cc diff --git a/Makefile b/Makefile index cca1d413443..493c3e6ae57 100644 --- a/Makefile +++ b/Makefile @@ -1146,6 +1146,7 @@ interop_client: $(BINDIR)/$(CONFIG)/interop_client interop_server: $(BINDIR)/$(CONFIG)/interop_server interop_test: $(BINDIR)/$(CONFIG)/interop_test json_run_localhost: $(BINDIR)/$(CONFIG)/json_run_localhost +memory_test: $(BINDIR)/$(CONFIG)/memory_test metrics_client: $(BINDIR)/$(CONFIG)/metrics_client mock_test: $(BINDIR)/$(CONFIG)/mock_test noop-benchmark: $(BINDIR)/$(CONFIG)/noop-benchmark @@ -1564,6 +1565,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/interop_server \ $(BINDIR)/$(CONFIG)/interop_test \ $(BINDIR)/$(CONFIG)/json_run_localhost \ + $(BINDIR)/$(CONFIG)/memory_test \ $(BINDIR)/$(CONFIG)/metrics_client \ $(BINDIR)/$(CONFIG)/mock_test \ $(BINDIR)/$(CONFIG)/noop-benchmark \ @@ -1682,6 +1684,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/interop_server \ $(BINDIR)/$(CONFIG)/interop_test \ $(BINDIR)/$(CONFIG)/json_run_localhost \ + $(BINDIR)/$(CONFIG)/memory_test \ $(BINDIR)/$(CONFIG)/metrics_client \ $(BINDIR)/$(CONFIG)/mock_test \ $(BINDIR)/$(CONFIG)/noop-benchmark \ @@ -2051,6 +2054,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/health_service_end2end_test || ( echo test health_service_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing interop_test" $(Q) $(BINDIR)/$(CONFIG)/interop_test || ( echo test interop_test failed ; exit 1 ) + $(E) "[RUN] Testing memory_test" + $(Q) $(BINDIR)/$(CONFIG)/memory_test || ( echo test memory_test failed ; exit 1 ) $(E) "[RUN] Testing mock_test" $(Q) $(BINDIR)/$(CONFIG)/mock_test || ( echo test mock_test failed ; exit 1 ) $(E) "[RUN] Testing noop-benchmark" @@ -14793,6 +14798,49 @@ endif endif +MEMORY_TEST_SRC = \ + test/core/support/memory_test.cc \ + +MEMORY_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(MEMORY_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/memory_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/memory_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/memory_test: $(PROTOBUF_DEP) $(MEMORY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(MEMORY_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/memory_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/test/core/support/memory_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_memory_test: $(MEMORY_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(MEMORY_TEST_OBJS:.o=.dep) +endif +endif + + METRICS_CLIENT_SRC = \ $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc \ test/cpp/interop/metrics_client.cc \ diff --git a/build.yaml b/build.yaml index 6a239829283..38c83d23e1d 100644 --- a/build.yaml +++ b/build.yaml @@ -89,6 +89,7 @@ filegroups: - src/core/lib/support/backoff.h - src/core/lib/support/block_annotate.h - src/core/lib/support/env.h + - src/core/lib/support/memory.h - src/core/lib/support/mpscq.h - src/core/lib/support/murmur_hash.h - src/core/lib/support/spinlock.h @@ -3810,6 +3811,20 @@ targets: - mac - linux - posix +- name: memory_test + gtest: true + build: test + language: c++ + src: + - test/core/support/memory_test.cc + deps: + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + uses: + - grpc++_test - name: metrics_client build: test run: false diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 1b6771d54f9..a4903b364f1 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -201,6 +201,7 @@ Pod::Spec.new do |s| 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', + 'src/core/lib/support/memory.h', 'src/core/lib/support/mpscq.h', 'src/core/lib/support/murmur_hash.h', 'src/core/lib/support/spinlock.h', @@ -697,6 +698,7 @@ Pod::Spec.new do |s| 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', + 'src/core/lib/support/memory.h', 'src/core/lib/support/mpscq.h', 'src/core/lib/support/murmur_hash.h', 'src/core/lib/support/spinlock.h', diff --git a/grpc.gemspec b/grpc.gemspec index 94d1316af34..81f85336755 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -86,6 +86,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/support/backoff.h ) s.files += %w( src/core/lib/support/block_annotate.h ) s.files += %w( src/core/lib/support/env.h ) + s.files += %w( src/core/lib/support/memory.h ) s.files += %w( src/core/lib/support/mpscq.h ) s.files += %w( src/core/lib/support/murmur_hash.h ) s.files += %w( src/core/lib/support/spinlock.h ) diff --git a/package.xml b/package.xml index f43fc69c513..b3150abf60e 100644 --- a/package.xml +++ b/package.xml @@ -95,6 +95,7 @@ + diff --git a/src/core/lib/support/memory.h b/src/core/lib/support/memory.h new file mode 100644 index 00000000000..3e00fb2011d --- /dev/null +++ b/src/core/lib/support/memory.h @@ -0,0 +1,74 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_SUPPORT_MEMORY_H +#define GRPC_SUPPORT_MEMORY_H + +#include + +#include +#include + +namespace grpc_core { + +// Alternative to new, since we cannot use it (for fear of libstdc++) +template +inline T* New(Args&&... args) { + void* p = gpr_malloc(sizeof(T)); + return new (p) T(std::forward(args)...); +} + +// Alternative to delete, since we cannot use it (for fear of libstdc++) +template +inline void Delete(T* p) { + p->~T(); + gpr_free(p); +} + +template +class DefaultDelete { + public: + void operator()(T* p) { Delete(p); } +}; + +template +using UniquePtr = std::unique_ptr>; + +template +inline UniquePtr MakeUnique(Args&&... args) { + return UniquePtr(New(std::forward(args)...)); +} + +} // namespace grpc_core + +#endif /* GRPC_SUPPORT_NEW_H */ diff --git a/test/core/support/memory_test.cc b/test/core/support/memory_test.cc new file mode 100644 index 00000000000..9e20e48b0cc --- /dev/null +++ b/test/core/support/memory_test.cc @@ -0,0 +1,76 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/lib/support/memory.h" +#include +#include "test/core/util/test_config.h" + +namespace grpc_core { +namespace testing { + +struct Foo { + Foo(int p, int q) : a(p), b(q) {} + int a; + int b; +}; + +TEST(MemoryTest, NewDeleteTest) { Delete(New()); } + +TEST(MemoryTest, NewDeleteWithArgTest) { + int* i = New(42); + EXPECT_EQ(42, *i); + Delete(i); +} + +TEST(MemoryTest, NewDeleteWithArgsTest) { + Foo* p = New(1, 2); + EXPECT_EQ(1, p->a); + EXPECT_EQ(2, p->b); + Delete(p); +} + +TEST(MemoryTest, MakeUniqueTest) { MakeUnique(); } + +TEST(MemoryTest, MakeUniqueWithArgTest) { + auto i = MakeUnique(42); + EXPECT_EQ(42, *i); +} + +} // namespace testing +} // namespace grpc_core + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 4b0fb700bb5..3b1b9cf0cd8 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1265,6 +1265,7 @@ src/core/lib/support/log_android.c \ src/core/lib/support/log_linux.c \ src/core/lib/support/log_posix.c \ src/core/lib/support/log_windows.c \ +src/core/lib/support/memory.h \ src/core/lib/support/mpscq.c \ src/core/lib/support/mpscq.h \ src/core/lib/support/murmur_hash.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 6557ecb7695..70a4a611430 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -3324,6 +3324,25 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_test", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c++", + "name": "memory_test", + "src": [ + "test/core/support/memory_test.cc" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -7347,6 +7366,7 @@ "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", + "src/core/lib/support/memory.h", "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.h", "src/core/lib/support/spinlock.h", @@ -7414,6 +7434,7 @@ "src/core/lib/support/log_linux.c", "src/core/lib/support/log_posix.c", "src/core/lib/support/log_windows.c", + "src/core/lib/support/memory.h", "src/core/lib/support/mpscq.c", "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.c", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 677407fe339..034576dd3f9 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -3371,6 +3371,28 @@ "posix" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "memory_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj b/vsprojects/vcxproj/gpr/gpr.vcxproj index 32dedee55d3..763cd1db6db 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj @@ -192,6 +192,7 @@ + diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters index 94699c8ff8c..4c4df99d436 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters @@ -272,6 +272,9 @@ src\core\lib\support + + src\core\lib\support + src\core\lib\support diff --git a/vsprojects/vcxproj/test/memory_test/memory_test.vcxproj b/vsprojects/vcxproj/test/memory_test/memory_test.vcxproj new file mode 100644 index 00000000000..1f4c1136ec8 --- /dev/null +++ b/vsprojects/vcxproj/test/memory_test/memory_test.vcxproj @@ -0,0 +1,204 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {0B674E04-7F49-A76B-3FF6-989D751B9AA4} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + memory_test + static + Debug + static + Debug + + + memory_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/memory_test/memory_test.vcxproj.filters b/vsprojects/vcxproj/test/memory_test/memory_test.vcxproj.filters new file mode 100644 index 00000000000..01a89346d2f --- /dev/null +++ b/vsprojects/vcxproj/test/memory_test/memory_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\support + + + + + + {80245c10-56a8-a6ec-0abc-8125f4271d38} + + + {eb61342c-1b95-756a-8b70-42aeb2a55f59} + + + {8c8dfaee-c0b7-e843-c50e-427448fe1eb9} + + + + From 3bf289d0860d9fe22ad24b652d6ee7e47a8a275b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 31 Mar 2017 14:32:51 -0700 Subject: [PATCH 02/60] Begin converting lame_client.c to C++ --- .../surface/{lame_client.c => lame_client.cc} | 62 +++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) rename src/core/lib/surface/{lame_client.c => lame_client.cc} (86%) diff --git a/src/core/lib/surface/lame_client.c b/src/core/lib/surface/lame_client.cc similarity index 86% rename from src/core/lib/surface/lame_client.c rename to src/core/lib/surface/lame_client.cc index 0c408aa2881..251b24d670c 100644 --- a/src/core/lib/surface/lame_client.c +++ b/src/core/lib/surface/lame_client.cc @@ -31,39 +31,49 @@ * */ -#include "src/core/lib/surface/lame_client.h" - #include #include +#include #include #include + +extern "C" { #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/support/string.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/call.h" #include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/lame_client.h" #include "src/core/lib/transport/static_metadata.h" +} + +namespace grpc_core { -typedef struct { +namespace { + +struct CallData { grpc_linked_mdelem status; grpc_linked_mdelem details; - gpr_atm filled_metadata; -} call_data; + std::atomic filled_metadata; +}; -typedef struct { +struct ChannelData { grpc_status_code error_code; const char *error_message; -} channel_data; +}; static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_metadata_batch *mdb) { - call_data *calld = elem->call_data; - if (!gpr_atm_no_barrier_cas(&calld->filled_metadata, 0, 1)) { + CallData *calld = static_cast(elem->call_data); + bool expected = false; + if (!calld->filled_metadata.compare_exchange_strong( + expected, true, std::memory_order_relaxed, + std::memory_order_relaxed)) { return; } - channel_data *chand = elem->channel_data; + ChannelData *chand = static_cast(elem->channel_data); char tmp[GPR_LTOA_MIN_BUFSIZE]; gpr_ltoa(chand->error_code, tmp); calld->status.md = grpc_mdelem_from_slices( @@ -83,7 +93,6 @@ static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, static void lame_start_transport_stream_op(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, grpc_transport_stream_op *op) { - GRPC_CALL_LOG_OP(GPR_INFO, elem, op); if (op->recv_initial_metadata != NULL) { fill_metadata(exec_ctx, elem, op->recv_initial_metadata); } else if (op->recv_trailing_metadata != NULL) { @@ -125,8 +134,6 @@ static void lame_start_transport_op(grpc_exec_ctx *exec_ctx, static grpc_error *init_call_elem(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, const grpc_call_element_args *args) { - call_data *calld = elem->call_data; - gpr_atm_no_barrier_store(&calld->filled_metadata, 0); return GRPC_ERROR_NONE; } @@ -147,18 +154,22 @@ static grpc_error *init_channel_elem(grpc_exec_ctx *exec_ctx, static void destroy_channel_elem(grpc_exec_ctx *exec_ctx, grpc_channel_element *elem) {} -const grpc_channel_filter grpc_lame_filter = { - lame_start_transport_stream_op, - lame_start_transport_op, - sizeof(call_data), - init_call_elem, +} // namespace + +} // namespace grpc_core + +extern "C" const grpc_channel_filter grpc_lame_filter = { + grpc_core::lame_start_transport_stream_op, + grpc_core::lame_start_transport_op, + sizeof(grpc_core::CallData), + grpc_core::init_call_elem, grpc_call_stack_ignore_set_pollset_or_pollset_set, - destroy_call_elem, - sizeof(channel_data), - init_channel_elem, - destroy_channel_elem, - lame_get_peer, - lame_get_channel_info, + grpc_core::destroy_call_elem, + sizeof(grpc_core::ChannelData), + grpc_core::init_channel_elem, + grpc_core::destroy_channel_elem, + grpc_core::lame_get_peer, + grpc_core::lame_get_channel_info, "lame-client", }; @@ -169,7 +180,6 @@ grpc_channel *grpc_lame_client_channel_create(const char *target, const char *error_message) { grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; grpc_channel_element *elem; - channel_data *chand; grpc_channel *channel = grpc_channel_create(&exec_ctx, target, NULL, GRPC_CLIENT_LAME_CHANNEL, NULL); elem = grpc_channel_stack_element(grpc_channel_get_channel_stack(channel), 0); @@ -178,7 +188,7 @@ grpc_channel *grpc_lame_client_channel_create(const char *target, "error_message=%s)", 3, (target, (int)error_code, error_message)); GPR_ASSERT(elem->filter == &grpc_lame_filter); - chand = (channel_data *)elem->channel_data; + auto chand = static_cast(elem->channel_data); chand->error_code = error_code; chand->error_message = error_message; grpc_exec_ctx_finish(&exec_ctx); From 7cbb4feff91ca0e8cee86688c5a613d217e3ac21 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 2 Apr 2017 10:03:19 -0700 Subject: [PATCH 03/60] Allow custom deletion policies on grpc_core::UniquePtr<> --- src/core/lib/support/memory.h | 4 ++-- test/core/support/memory_test.cc | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/core/lib/support/memory.h b/src/core/lib/support/memory.h index 3e00fb2011d..bd56ca39394 100644 --- a/src/core/lib/support/memory.h +++ b/src/core/lib/support/memory.h @@ -61,8 +61,8 @@ class DefaultDelete { void operator()(T* p) { Delete(p); } }; -template -using UniquePtr = std::unique_ptr>; +template > +using UniquePtr = std::unique_ptr; template inline UniquePtr MakeUnique(Args&&... args) { diff --git a/test/core/support/memory_test.cc b/test/core/support/memory_test.cc index 9e20e48b0cc..8db316a4238 100644 --- a/test/core/support/memory_test.cc +++ b/test/core/support/memory_test.cc @@ -66,6 +66,19 @@ TEST(MemoryTest, MakeUniqueWithArgTest) { EXPECT_EQ(42, *i); } +TEST(MemoryTest, UniquePtrWithCustomDeleter) { + int n = 0; + class IncrementingDeleter { + public: + void operator()(int* p) { ++*p; } + }; + { + UniquePtr p(&n); + EXPECT_EQ(0, n); + } + EXPECT_EQ(1, n); +} + } // namespace testing } // namespace grpc_core From 36f9bb0edf3d46e0edcfb21ff4e234b8f2e899b6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 2 Apr 2017 16:45:26 -0700 Subject: [PATCH 04/60] Fix include guards --- src/core/lib/support/memory.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/lib/support/memory.h b/src/core/lib/support/memory.h index bd56ca39394..6eff94eff78 100644 --- a/src/core/lib/support/memory.h +++ b/src/core/lib/support/memory.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_SUPPORT_MEMORY_H -#define GRPC_SUPPORT_MEMORY_H +#ifndef GRPC_CORE_LIB_SUPPORT_MEMORY_H +#define GRPC_CORE_LIB_SUPPORT_MEMORY_H #include @@ -71,4 +71,4 @@ inline UniquePtr MakeUnique(Args&&... args) { } // namespace grpc_core -#endif /* GRPC_SUPPORT_NEW_H */ +#endif /* GRPC_CORE_LIB_SUPPORT_MEMORY_H */ From ec581cd5903d1a1fd129d647daa289a081b2dd1a Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Fri, 7 Apr 2017 12:50:33 -0700 Subject: [PATCH 05/60] Update TSI with new non-blocking TSI interfaces. --- Makefile | 3 + src/core/tsi/fake_transport_security.c | 1 + src/core/tsi/ssl_transport_security.c | 1 + src/core/tsi/transport_security.c | 44 +++ src/core/tsi/transport_security.h | 22 ++ src/core/tsi/transport_security_adapter.c | 230 ++++++++++++++++ src/core/tsi/transport_security_adapter.h | 55 ++++ src/core/tsi/transport_security_interface.h | 279 ++++++++++++++++---- test/core/tsi/transport_security_test.c | 2 + 9 files changed, 585 insertions(+), 52 deletions(-) create mode 100644 src/core/tsi/transport_security_adapter.c create mode 100644 src/core/tsi/transport_security_adapter.h diff --git a/Makefile b/Makefile index 91a74f354e8..cbf38709911 100644 --- a/Makefile +++ b/Makefile @@ -2987,6 +2987,7 @@ LIBGRPC_SRC = \ src/core/tsi/fake_transport_security.c \ src/core/tsi/ssl_transport_security.c \ src/core/tsi/transport_security.c \ + src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \ src/core/ext/filters/client_channel/channel_connectivity.c \ @@ -3330,6 +3331,7 @@ LIBGRPC_CRONET_SRC = \ src/core/tsi/fake_transport_security.c \ src/core/tsi/ssl_transport_security.c \ src/core/tsi/transport_security.c \ + src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/client/chttp2_connector.c \ src/core/ext/filters/load_reporting/load_reporting.c \ src/core/ext/filters/load_reporting/load_reporting_filter.c \ @@ -19257,6 +19259,7 @@ src/core/plugin_registry/grpc_plugin_registry.c: $(OPENSSL_DEP) src/core/tsi/fake_transport_security.c: $(OPENSSL_DEP) src/core/tsi/ssl_transport_security.c: $(OPENSSL_DEP) src/core/tsi/transport_security.c: $(OPENSSL_DEP) +src/core/tsi/transport_security_adapter.c: $(OPENSSL_DEP) src/cpp/client/cronet_credentials.cc: $(OPENSSL_DEP) src/cpp/client/secure_credentials.cc: $(OPENSSL_DEP) src/cpp/common/auth_property_iterator.cc: $(OPENSSL_DEP) diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index 822fad51cb3..1836beefc42 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -499,6 +499,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { fake_handshaker_extract_peer, fake_handshaker_create_frame_protector, fake_handshaker_destroy, + NULL, }; tsi_handshaker *tsi_create_fake_handshaker(int is_client) { diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index a0325cc1835..c4dba002f8b 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -1049,6 +1049,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { ssl_handshaker_extract_peer, ssl_handshaker_create_frame_protector, ssl_handshaker_destroy, + NULL, }; /* --- tsi_ssl_handshaker_factory common methods. --- */ diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index 67ebe1b1f30..71f82958043 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -73,6 +73,8 @@ const char *tsi_result_to_string(tsi_result result) { return "TSI_HANDSHAKE_IN_PROGRESS"; case TSI_OUT_OF_RESOURCES: return "TSI_OUT_OF_RESOURCES"; + case TSI_ASYNC: + return "TSI_ASYNC"; default: return "UNKNOWN"; } @@ -185,11 +187,53 @@ tsi_result tsi_handshaker_create_frame_protector( return result; } +tsi_result tsi_handshaker_next( + tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb *cb, void *user_data) { + if (self == NULL) return TSI_INVALID_ARGUMENT; + if (self->handshaker_result_created) return TSI_FAILED_PRECONDITION; + return self->vtable->next(self, received_bytes, received_bytes_size, + bytes_to_send, bytes_to_send_size, + handshaker_result, cb, user_data); +} + void tsi_handshaker_destroy(tsi_handshaker *self) { if (self == NULL) return; self->vtable->destroy(self); } +/* --- tsi_handshaker_result implementation. --- */ + +tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, + tsi_peer *peer) { + if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT; + memset(peer, 0, sizeof(tsi_peer)); + return self->vtable->extract_peer(self, peer); +} + +tsi_result tsi_handshaker_result_create_frame_protector( + tsi_handshaker_result *self, size_t *max_protected_frame_size, + tsi_frame_protector **protector) { + if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT; + return self->vtable->create_frame_protector(self, max_protected_frame_size, + protector); +} + +tsi_result tsi_handshaker_result_get_unused_bytes( + tsi_handshaker_result *self, unsigned char **bytes, size_t *bytes_size) { + if (self == NULL || bytes == NULL || bytes_size == NULL) { + return TSI_INVALID_ARGUMENT; + } + return self->vtable->get_unused_bytes(self, bytes, bytes_size); +} + +void tsi_handshaker_result_destroy(tsi_handshaker_result *self) { + if (self == NULL) return; + self->vtable->destroy(self); +} + /* --- tsi_peer implementation. --- */ tsi_peer_property tsi_init_peer_property(void) { diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index 491fa1a8bd6..552ee22061a 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -81,11 +81,33 @@ typedef struct { size_t *max_protected_frame_size, tsi_frame_protector **protector); void (*destroy)(tsi_handshaker *self); + tsi_result (*next)(tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, + tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb *cb, void *user_data); } tsi_handshaker_vtable; struct tsi_handshaker { const tsi_handshaker_vtable *vtable; int frame_protector_created; + int handshaker_result_created; +}; + +/* Base for tsi_handshaker_result implementations. + See transport_security_interface.h for documentation. */ +typedef struct { + tsi_result (*extract_peer)(tsi_handshaker_result *self, tsi_peer *peer); + tsi_result (*create_frame_protector)(tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, + tsi_frame_protector **protector); + tsi_result (*get_unused_bytes)(tsi_handshaker_result *self, + unsigned char **bytes, size_t *bytes_size); + void (*destroy)(tsi_handshaker_result *self); +} tsi_handshaker_result_vtable; + +struct tsi_handshaker_result { + const tsi_handshaker_result_vtable *vtable; }; /* Peer and property construction/destruction functions. */ diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c new file mode 100644 index 00000000000..9a7808c09c8 --- /dev/null +++ b/src/core/tsi/transport_security_adapter.c @@ -0,0 +1,230 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/tsi/transport_security_adapter.h" + +#include + +#include +#include +#include "src/core/tsi/transport_security.h" + +#define TSI_ADAPTER_INITIAL_BUFFER_SIZE 256 + +/* --- tsi_adapter_handshaker_result implementation ---*/ + +typedef struct { + tsi_handshaker_result base; + tsi_handshaker *handshaker; + unsigned char *unused_bytes; + size_t unused_bytes_size; +} tsi_adapter_handshaker_result; + +static tsi_result tsi_adapter_result_extract_peer(tsi_handshaker_result *self, + tsi_peer *peer) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + return tsi_handshaker_extract_peer(impl->handshaker, peer); +} + +static tsi_result tsi_adapter_result_create_frame_protector( + tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_frame_protector **protector) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + return tsi_handshaker_create_frame_protector( + impl->handshaker, max_output_protected_frame_size, protector); +} + +static tsi_result tsi_adapter_result_get_unused_bytes( + tsi_handshaker_result *self, unsigned char **bytes, size_t *byte_size) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + *bytes = impl->unused_bytes; + *byte_size = impl->unused_bytes_size; + return TSI_OK; +} + +static void tsi_adapter_result_destroy(tsi_handshaker_result *self) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + if (impl->unused_bytes != NULL) { + gpr_free(impl->unused_bytes); + } + gpr_free(self); +} + +static const tsi_handshaker_result_vtable result_vtable = { + tsi_adapter_result_extract_peer, tsi_adapter_result_create_frame_protector, + tsi_adapter_result_get_unused_bytes, tsi_adapter_result_destroy, +}; + +tsi_result tsi_adapter_create_handshaker_result( + tsi_handshaker *handshaker, const unsigned char *unused_bytes, + size_t unused_bytes_size, tsi_handshaker_result **handshaker_result) { + if (handshaker == NULL || (unused_bytes_size > 0 && unused_bytes == NULL)) { + return TSI_INVALID_ARGUMENT; + } + tsi_adapter_handshaker_result *impl = gpr_zalloc(sizeof(*impl)); + impl->base.vtable = &result_vtable; + impl->handshaker = handshaker; + impl->unused_bytes_size = unused_bytes_size; + if (unused_bytes_size > 0) { + impl->unused_bytes = gpr_malloc(unused_bytes_size); + memcpy(impl->unused_bytes, unused_bytes, unused_bytes_size); + } else { + impl->unused_bytes = NULL; + } + *handshaker_result = &impl->base; + return TSI_OK; +} + +/* --- tsi_adapter_handshaker implementation ---*/ + +typedef struct { + tsi_handshaker base; + tsi_handshaker *wrapped; + unsigned char *adapter_buffer; + size_t adapter_buffer_size; +} tsi_adapter_handshaker; + +tsi_result tsi_adapter_get_bytes_to_send_to_peer(tsi_handshaker *self, + unsigned char *bytes, + size_t *bytes_size) { + return tsi_handshaker_get_bytes_to_send_to_peer( + tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); +} + +tsi_result tsi_adapter_process_bytes_from_peer(tsi_handshaker *self, + const unsigned char *bytes, + size_t *bytes_size) { + return tsi_handshaker_process_bytes_from_peer( + tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); +} + +tsi_result tsi_adapter_get_result(tsi_handshaker *self) { + return tsi_handshaker_get_result(tsi_adapter_handshaker_get_wrapped(self)); +} + +tsi_result tsi_adapter_extract_peer(tsi_handshaker *self, tsi_peer *peer) { + return tsi_handshaker_extract_peer(tsi_adapter_handshaker_get_wrapped(self), + peer); +} + +tsi_result tsi_adapter_create_frame_protector(tsi_handshaker *self, + size_t *max_protected_frame_size, + tsi_frame_protector **protector) { + return tsi_handshaker_create_frame_protector( + tsi_adapter_handshaker_get_wrapped(self), max_protected_frame_size, + protector); +} + +void tsi_adapter_destroy(tsi_handshaker *self) { + tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self; + tsi_handshaker_destroy(impl->wrapped); + gpr_free(impl->adapter_buffer); + gpr_free(self); +} + +tsi_result tsi_adapter_next( + tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb *cb, void *user_data) { + /* Input sanity check. */ + if ((received_bytes_size > 0 && received_bytes == NULL) || + bytes_to_send == NULL || bytes_to_send_size == NULL || + handshaker_result == NULL) { + return TSI_INVALID_ARGUMENT; + } + + /* If there are received bytes, process them first. */ + tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self; + tsi_result status = TSI_OK; + size_t bytes_consumed = received_bytes_size; + if (received_bytes_size > 0) { + status = tsi_handshaker_process_bytes_from_peer( + impl->wrapped, received_bytes, &bytes_consumed); + if (status != TSI_OK) return status; + } + + /* Get bytes to send to the peer, if available. */ + size_t offset = 0; + do { + size_t to_send_size = impl->adapter_buffer_size - offset; + status = tsi_handshaker_get_bytes_to_send_to_peer( + impl->wrapped, impl->adapter_buffer + offset, &to_send_size); + offset += to_send_size; + if (status == TSI_INCOMPLETE_DATA) { + impl->adapter_buffer_size *= 2; + impl->adapter_buffer = + gpr_realloc(impl->adapter_buffer, impl->adapter_buffer_size); + } + } while (status == TSI_INCOMPLETE_DATA); + if (status != TSI_OK) return status; + *bytes_to_send = impl->adapter_buffer; + *bytes_to_send_size = offset; + + /* If handshake completes, create tsi_handshaker_result. */ + if (!tsi_handshaker_is_in_progress(impl->wrapped)) { + size_t unused_bytes_size = received_bytes_size - bytes_consumed; + const unsigned char *unused_bytes = + unused_bytes_size == 0 ? NULL : received_bytes + bytes_consumed; + return tsi_adapter_create_handshaker_result( + impl->wrapped, unused_bytes, unused_bytes_size, handshaker_result); + } + *handshaker_result = NULL; + return TSI_OK; +} + +static const tsi_handshaker_vtable handshaker_vtable = { + tsi_adapter_get_bytes_to_send_to_peer, + tsi_adapter_process_bytes_from_peer, + tsi_adapter_get_result, + tsi_adapter_extract_peer, + tsi_adapter_create_frame_protector, + tsi_adapter_destroy, + tsi_adapter_next, +}; + +tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped) { + GPR_ASSERT(wrapped != NULL); + tsi_adapter_handshaker *impl = gpr_zalloc(sizeof(*impl)); + impl->base.vtable = &handshaker_vtable; + impl->wrapped = wrapped; + impl->adapter_buffer_size = TSI_ADAPTER_INITIAL_BUFFER_SIZE; + impl->adapter_buffer = gpr_malloc(impl->adapter_buffer_size); + return &impl->base; +} + +tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter) { + if (adapter == NULL) return NULL; + tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)adapter; + return impl->wrapped; +} diff --git a/src/core/tsi/transport_security_adapter.h b/src/core/tsi/transport_security_adapter.h new file mode 100644 index 00000000000..de5aaefd5f1 --- /dev/null +++ b/src/core/tsi/transport_security_adapter.h @@ -0,0 +1,55 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H +#define GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H + +#include "src/core/tsi/transport_security_interface.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Create a tsi handshaker that takes an implementation of old interface and + converts into an implementation of new interface. + Ownership of input tsi_handshaker is transferred to this new adapter. */ +tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped); + +/* Given a tsi adapter handshaker, return the original wrapped handshaker. */ +tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter); + +#ifdef __cplusplus +} +#endif + +#endif // GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index caed43eac44..eaa02bf7f5f 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -56,7 +56,8 @@ typedef enum { TSI_NOT_FOUND = 9, TSI_PROTOCOL_FAILURE = 10, TSI_HANDSHAKE_IN_PROGRESS = 11, - TSI_OUT_OF_RESOURCES = 12 + TSI_OUT_OF_RESOURCES = 12, + TSI_ASYNC = 13 } tsi_result; typedef enum { @@ -208,76 +209,203 @@ typedef struct { /* Destructs the tsi_peer object. */ void tsi_peer_destruct(tsi_peer *self); +/* --- tsi_handshaker_result object --- + + This object contains all necessary handshake results and data such as peer + info, negotiated keys, unused handshake bytes, when the handshake completes. + Implementations of this object must be thread compatible. */ + +typedef struct tsi_handshaker_result tsi_handshaker_result; + +/* This method extracts tsi peer. It returns TSI_OK assuming there is no fatal + error. + The caller is responsible for destructing the peer. */ +tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, + tsi_peer *peer); + +/* This method creates a tsi_frame_protector object. It returns TSI_OK assuming + there is no fatal error. + The caller is responsible for destroying the protector. */ +tsi_result tsi_handshaker_result_create_frame_protector( + tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_frame_protector **protector); + +/* This method returns the unused bytes from the handshake. It returns TSI_OK + assuming there is no fatal error. + The caller should not free the bytes. */ +tsi_result tsi_handshaker_result_get_unused_bytes(tsi_handshaker_result *self, + unsigned char **bytes, + size_t *byte_size); + +/* This method releases the tsi_handshaker_handshaker object. After this method + is called, no other method can be called on the object. */ +void tsi_handshaker_result_destroy(tsi_handshaker_result *self); + /* --- tsi_handshaker objects ---- Implementations of this object must be thread compatible. - A typical usage of this object would be: + ------------------------------------------------------------------------ + + A typical usage of the synchronous TSI handshaker would be: ------------------------------------------------------------------------ - tsi_result result = TSI_OK; + tsi_result status = TSI_OK; unsigned char buf[4096]; - size_t buf_offset; - size_t buf_size; + const size_t buf_size = 4906; + size_t bytes_received_size = 0; + unsigned char *bytes_to_send = NULL; + size_t bytes_to_send_size = 0; + tsi_handshaker_result *result = NULL; + while (1) { - // See if we need to send some bytes to the peer. - do { - size_t buf_size_to_send = sizeof(buf); - result = tsi_handshaker_get_bytes_to_send_to_peer(handshaker, buf, - &buf_size_to_send); - if (buf_size_to_send > 0) send_bytes_to_peer(buf, buf_size_to_send); - } while (result == TSI_INCOMPLETE_DATA); - if (result != TSI_OK) return result; - if (!tsi_handshaker_is_in_progress(handshaker)) break; - - do { - // Read bytes from the peer. - buf_size = sizeof(buf); - buf_offset = 0; - read_bytes_from_peer(buf, &buf_size); - if (buf_size == 0) break; - - // Process the bytes from the peer. We have to be careful as these bytes - // may contain non-handshake data (protected data). If this is the case, - // we will exit from the loop with buf_size > 0. - size_t consumed_by_handshaker = buf_size; - result = tsi_handshaker_process_bytes_from_peer( - handshaker, buf, &consumed_by_handshaker); - buf_size -= consumed_by_handshaker; - buf_offset += consumed_by_handshaker; - } while (result == TSI_INCOMPLETE_DATA); - - if (result != TSI_OK) return result; - if (!tsi_handshaker_is_in_progress(handshaker)) break; + status = tsi_handshaker_next( + handshaker, buf, bytes_received_size, + &bytes_to_send, &bytes_to_send_size, &result, NULL, NULL); + if (status == TSI_INCOMPLETE_DATA) { + // Need more data from the peer. + bytes_received_size = buf_size; + read_bytes_from_peer(buf, &bytes_received_size); + continue; + } + if (status != TSI_OK) return status; + if (bytes_to_send_size > 0) { + send_bytes_to_peer(bytes_to_send, bytes_to_send_size); + } + if (result != NULL) break; + bytes_received_size = buf_size; + read_bytes_from_peer(buf, &bytes_received_size); } // Check the Peer. tsi_peer peer; - do { - result = tsi_handshaker_extract_peer(handshaker, &peer); - if (result != TSI_OK) break; - result = check_peer(&peer); - } while (0); + status = tsi_handshaker_result_extract_peer(result, &peer); + if (status != TSI_OK) return status; + status = check_peer(&peer); tsi_peer_destruct(&peer); - if (result != TSI_OK) return result; + if (status != TSI_OK) return status; // Create the protector. tsi_frame_protector* protector = NULL; - result = tsi_handshaker_create_frame_protector(handshaker, NULL, - &protector); - if (result != TSI_OK) return result; + status = tsi_handshaker_result_create_frame_protector(result, NULL, + &protector); + if (status != TSI_OK) return status; // Do not forget to unprotect outstanding data if any. - if (buf_size > 0) { - result = tsi_frame_protector_unprotect(protector, buf + buf_offset, - buf_size, ..., ...); + unsigned char *unused_bytes = NULL; + size_t unused_bytes_size = 0; + status = tsi_handshaker_result_get_unused_byte(result, &unused_bytes, + &unused_bytes_size); + if (status != TSI_OK) return status; + if (unused_bytes_size > 0) { + status = tsi_frame_protector_unprotect(protector, unused_bytes, + unused_bytes_size, ..., ...); .... } ... + ------------------------------------------------------------------------ + + A typical usage of the new TSI would be as follows, supporting both + synchronous and asynchrnous TSI handshaker implementations: + + ------------------------------------------------------------------------ + + typedef struct { + tsi_handshaker *handshaker; + tsi_handshaker_result *handshaker_result; + unsigned char *handshake_buffer; + size_t handshake_buffer_size; + ... + } security_handshaker; + + void do_handshake(security_handshaker *h, ...) { + // Start the handshake by the calling do_handshake_next. + do_handshake_next(h, NULL, 0); + ... + } + + // This method is a wrapper of the callback function to execute when + // tsi_handshaker_next finishes. It is passed to tsi_handshaker_next as + // the callback function. + void on_handshake_next_done_wrapper( + tsi_result status, void *user_data, const unsigned char *bytes_to_send, + size_t bytes_to_send_size, const tsi_handshaker_result *result) { + security_handshaker *h = (security_handshaker *)user_data; + on_handshake_next_done(h, status, bytes_to_send, + bytes_to_send_size, result); + } + + // This method is the callback function when there are data received from + // the peer. This method will read bytes into the handshake buffer and call + // do_handshake_next. + void on_handshake_data_received_from_peer(void *user_data) { + security_handshaker *h = (security_handshaker *)user_data; + size_t bytes_received_size = h->handshake_buffer_size; + read_bytes_from_peer(h->handshake_buffer, &bytes_received_size); + do_handshake_next(h, h->handshake_buffer, bytes_received_size); + } + + // This method processes a step of handshake, calling tsi_handshaker_next. + void do_handshake_next(security_handshaker *h, + const unsigned char* bytes_received, + size_t bytes_received_size) { + tsi_result status = TSI_OK; + unsigned char *bytes_to_send = NULL; + size_t bytes_to_send_size = 0; + tsi_handshaker_result *result = NULL; + status = tsi_handshaker_next( + handshaker, bytes_received, bytes_received_size, &bytes_to_send, + &bytes_to_send_size, &result, on_handshake_next_done_wrapper, h); + // If TSI handshaker is asynchronous, on_handshake_next_done will be + // called during the execution of the callback function. + if (status == TSI_ASYNC) return; + on_handshake_next_done(h, status, bytes_to_send, + bytes_to_send_size, result); + } + + // This is the real function to execute after tsi_handshaker_next. + void on_handshake_next_done( + security_handshaker *h, tsi_result status, + const unsigned char *bytes_to_send, size_t bytes_to_send_size, + const tsi_handshaker_result *result) { + if (status == TSI_INCOMPLETE_DATA) { + // Schedule an asynchronous read from the peer. If handshake data are + // received, on_handshake_data_received_from_peer will be called. + async_read_from_peer(..., ..., on_handshake_data_received_from_peer); + return; + } + if (status != TSI_OK) return; + + if (bytes_to_send_size > 0) { + send_bytes_to_peer(bytes_to_send, bytes_to_send_size); + } + + if (result != NULL) { + // Handshake completed. + h->result = result; + // Check the Peer. + tsi_peer peer; + status = tsi_handshaker_result_extract_peer(result, &peer); + if (status != TSI_OK) return status; + status = check_peer(&peer); + tsi_peer_destruct(&peer); + if (status != TSI_OK) return; + + // Create the protector. + tsi_frame_protector* protector = NULL; + status = tsi_handshaker_result_create_frame_protector(result, NULL, + &protector); + if (status != TSI_OK) return; + + // Do not forget to unprotect outstanding data if any. + .... + } + } ------------------------------------------------------------------------ */ typedef struct tsi_handshaker tsi_handshaker; -/* Gets bytes that need to be sent to the peer. +/* TO BE DEPRECATED SOON. + Gets bytes that need to be sent to the peer. - bytes is the buffer that will be written with the data to be sent to the peer. - bytes_size is an input/output parameter specifying the capacity of the @@ -292,7 +420,8 @@ tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, unsigned char *bytes, size_t *bytes_size); -/* Processes bytes received from the peer. +/* TO BE DEPRECATED SOON. + Processes bytes received from the peer. - bytes is the buffer containing the data. - bytes_size is an input/output parameter specifying the size of the data as input and the number of bytes consumed as output. @@ -305,24 +434,28 @@ tsi_result tsi_handshaker_process_bytes_from_peer(tsi_handshaker *self, const unsigned char *bytes, size_t *bytes_size); -/* Gets the result of the handshaker. +/* TO BE DEPRECATED SOON. + Gets the result of the handshaker. Returns TSI_OK if the hanshake completed successfully and there has been no errors. Returns TSI_HANDSHAKE_IN_PROGRESS if the handshaker is not done yet but no error has been encountered so far. Otherwise the handshaker failed with the returned error. */ tsi_result tsi_handshaker_get_result(tsi_handshaker *self); -/* Returns 1 if the handshake is in progress, 0 otherwise. */ +/* TO BE DEPRECATED SOON. + Returns 1 if the handshake is in progress, 0 otherwise. */ #define tsi_handshaker_is_in_progress(h) \ (tsi_handshaker_get_result((h)) == TSI_HANDSHAKE_IN_PROGRESS) -/* This method may return TSI_FAILED_PRECONDITION if +/* TO BE DEPRECATED SOON. + This method may return TSI_FAILED_PRECONDITION if tsi_handshaker_is_in_progress returns 1, it returns TSI_OK otherwise assuming the handshaker is not in a fatal error state. The caller is responsible for destructing the peer. */ tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer); -/* This method creates a tsi_frame_protector object after the handshake phase +/* TO BE DEPRECATED SOON. + This method creates a tsi_frame_protector object after the handshake phase is done. After this method has been called successfully, the only method that can be called on this object is Destroy. - max_output_protected_frame_size is an input/output parameter specifying the @@ -342,10 +475,52 @@ tsi_result tsi_handshaker_create_frame_protector( tsi_handshaker *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector); +/* Callback function definition for tsi_handshaker_next. + - status indicates the status of the next operation. + - user_data is the argument to callback function passed from the caller. + - bytes_to_send is the data buffer to be sent to the peer. + - bytes_to_send_size is the size of data buffer to be sent to the peer. + - handshaker_result is the result of handshake when the handshake completes, + is NULL otherwise. */ +typedef void (*tsi_handshaker_on_next_done_cb)( + tsi_result status, void *user_data, const unsigned char *bytes_to_send, + size_t bytes_to_send_size, const tsi_handshaker_result *handshaker_result); + +/* Conduct a next step of the handshake. + - received_bytes is the buffer containing the data received from the peer. + - received_bytes_size is the size of the data received from the peer. + - bytes_to_send is the data buffer to be sent to the peer. + - bytes_to_send_size is the size of data buffer to be sent to the peer. + - handshaker_result is the result of handshake if the handshake completes. + - cb is the callback function defined above. It can be NULL for synchronous + TSI handshaker implementation. + - user_data is the argument to callback function passed from the caller. + This method returns TSI_ASYNC if the TSI handshaker implementation is + asynchronous. It returns TSI_OK if the handshake completes or if there are + data to send to the peer, otherwise returns TSI_INCOMPLETE_DATA which + indicates that this method needs to be called again with more data from the + peer. In case of a fatal error in the handshake, another specific error code + is returned. + The caller is responsible for destroying the handshaker_result. However, the + caller should not free bytes_to_send, as the buffer is owned by the + tsi_handshaker object. */ +tsi_result tsi_handshaker_next( + tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb *cb, void *user_data); + /* This method releases the tsi_handshaker object. After this method is called, no other method can be called on the object. */ void tsi_handshaker_destroy(tsi_handshaker *self); +/* This method initializes the necessary shared objects used for tsi + implementation. */ +void tsi_init(); + +/* This method destroys the shared objects created by tsi_init. */ +void tsi_destroy(); + #ifdef __cplusplus } #endif diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index ee4a37c3141..4214407823a 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -376,6 +376,8 @@ static void test_handshaker_invalid_args(void) { TSI_INVALID_ARGUMENT); GPR_ASSERT(tsi_handshaker_get_bytes_to_send_to_peer(NULL, NULL, NULL) == TSI_INVALID_ARGUMENT); + GPR_ASSERT(tsi_handshaker_next(NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL) == + TSI_INVALID_ARGUMENT); } static void test_handshaker_invalid_state(void) { From 8ebb5443dee350d8740785461b4ac03534815778 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 7 Apr 2017 16:01:55 -0700 Subject: [PATCH 06/60] Update build.yaml --- CMakeLists.txt | 10 +++++----- Makefile | 10 +++++----- binding.gyp | 2 +- build.yaml | 2 +- config.m4 | 2 +- gRPC-Core.podspec | 2 +- grpc.gemspec | 2 +- package.xml | 2 +- src/python/grpcio/grpc_core_dependencies.py | 2 +- tools/doxygen/Doxyfile.core.internal | 2 +- tools/run_tests/generated/sources_and_headers.json | 2 +- vsprojects/vcxproj/grpc/grpc.vcxproj | 2 +- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 2 +- .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 2 +- .../grpc_test_util/grpc_test_util.vcxproj.filters | 2 +- vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 2 +- .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 2 +- 17 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f75231cfe14..063cdd370e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1016,7 +1016,7 @@ add_library(grpc src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c @@ -1338,7 +1338,7 @@ add_library(grpc_cronet src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c @@ -1647,7 +1647,7 @@ add_library(grpc_test_util src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c @@ -1908,7 +1908,7 @@ add_library(grpc_unsecure src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c @@ -2540,7 +2540,7 @@ add_library(grpc++_cronet src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c diff --git a/Makefile b/Makefile index 493c3e6ae57..cae6609c5cd 100644 --- a/Makefile +++ b/Makefile @@ -2922,7 +2922,7 @@ LIBGRPC_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ @@ -3242,7 +3242,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ @@ -3550,7 +3550,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ @@ -3783,7 +3783,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ @@ -4401,7 +4401,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ diff --git a/binding.gyp b/binding.gyp index 0023c240983..dd2ecb0850c 100644 --- a/binding.gyp +++ b/binding.gyp @@ -722,7 +722,7 @@ 'src/core/lib/surface/completion_queue.c', 'src/core/lib/surface/completion_queue_factory.c', 'src/core/lib/surface/event_string.c', - 'src/core/lib/surface/lame_client.c', + 'src/core/lib/surface/lame_client.cc', 'src/core/lib/surface/metadata_array.c', 'src/core/lib/surface/server.c', 'src/core/lib/surface/validate_metadata.c', diff --git a/build.yaml b/build.yaml index 38c83d23e1d..2aa1c236f88 100644 --- a/build.yaml +++ b/build.yaml @@ -393,7 +393,7 @@ filegroups: - src/core/lib/surface/completion_queue.c - src/core/lib/surface/completion_queue_factory.c - src/core/lib/surface/event_string.c - - src/core/lib/surface/lame_client.c + - src/core/lib/surface/lame_client.cc - src/core/lib/surface/metadata_array.c - src/core/lib/surface/server.c - src/core/lib/surface/validate_metadata.c diff --git a/config.m4 b/config.m4 index fe2011922a6..a3c078f7a78 100644 --- a/config.m4 +++ b/config.m4 @@ -190,7 +190,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index a4903b364f1..012da513dba 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -567,7 +567,7 @@ Pod::Spec.new do |s| 'src/core/lib/surface/completion_queue.c', 'src/core/lib/surface/completion_queue_factory.c', 'src/core/lib/surface/event_string.c', - 'src/core/lib/surface/lame_client.c', + 'src/core/lib/surface/lame_client.cc', 'src/core/lib/surface/metadata_array.c', 'src/core/lib/surface/server.c', 'src/core/lib/surface/validate_metadata.c', diff --git a/grpc.gemspec b/grpc.gemspec index 81f85336755..c4a5b7be5bb 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -483,7 +483,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/surface/completion_queue.c ) s.files += %w( src/core/lib/surface/completion_queue_factory.c ) s.files += %w( src/core/lib/surface/event_string.c ) - s.files += %w( src/core/lib/surface/lame_client.c ) + s.files += %w( src/core/lib/surface/lame_client.cc ) s.files += %w( src/core/lib/surface/metadata_array.c ) s.files += %w( src/core/lib/surface/server.c ) s.files += %w( src/core/lib/surface/validate_metadata.c ) diff --git a/package.xml b/package.xml index b3150abf60e..3d466d6b1e6 100644 --- a/package.xml +++ b/package.xml @@ -492,7 +492,7 @@ - + diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 9ae59c9df27..9c18b299837 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -184,7 +184,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/surface/completion_queue.c', 'src/core/lib/surface/completion_queue_factory.c', 'src/core/lib/surface/event_string.c', - 'src/core/lib/surface/lame_client.c', + 'src/core/lib/surface/lame_client.cc', 'src/core/lib/surface/metadata_array.c', 'src/core/lib/surface/server.c', 'src/core/lib/surface/validate_metadata.c', diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 3b1b9cf0cd8..dee289c7dda 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1326,7 +1326,7 @@ src/core/lib/surface/event_string.h \ src/core/lib/surface/init.c \ src/core/lib/surface/init.h \ src/core/lib/surface/init_secure.c \ -src/core/lib/surface/lame_client.c \ +src/core/lib/surface/lame_client.cc \ src/core/lib/surface/lame_client.h \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 70a4a611430..791066514f7 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7849,7 +7849,7 @@ "src/core/lib/surface/event_string.c", "src/core/lib/surface/event_string.h", "src/core/lib/surface/init.h", - "src/core/lib/surface/lame_client.c", + "src/core/lib/surface/lame_client.cc", "src/core/lib/surface/lame_client.h", "src/core/lib/surface/metadata_array.c", "src/core/lib/surface/server.c", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index ad46b8e9112..e5c2b30030f 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -720,7 +720,7 @@ - + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 7492847e3b4..e87cb6b670c 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -325,7 +325,7 @@ src\core\lib\surface - + src\core\lib\surface diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 1cfe6d627e8..0552cad44c9 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -561,7 +561,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 096d5c7e595..2d74b57fd75 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -382,7 +382,7 @@ src\core\lib\surface - + src\core\lib\surface diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index fe850562801..44cd17bebed 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -687,7 +687,7 @@ - + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 4bcd2371245..659843e72b0 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -328,7 +328,7 @@ src\core\lib\surface - + src\core\lib\surface From a0dacc3c4880a51dc3564a3c797ad91feb253295 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Fri, 7 Apr 2017 16:33:07 -0700 Subject: [PATCH 07/60] C++ build for python --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e0506464759..88dc54f8d0d 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ ENABLE_DOCUMENTATION_BUILD = os.environ.get( EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) if EXTRA_ENV_COMPILE_ARGS is None: - EXTRA_ENV_COMPILE_ARGS = '' + EXTRA_ENV_COMPILE_ARGS = '-std=c++11' if 'win32' in sys.platform and sys.version_info < (3, 5): # We use define flags here and don't directly add to DEFINE_MACROS below to # ensure that the expert user/builder has a way of turning it off (via the @@ -195,6 +195,7 @@ def cython_extensions_and_necessity(): extensions = [ _extension.Extension( name=module_name, + language='c++', sources=[module_file] + list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES), include_dirs=list(EXTENSION_INCLUDE_DIRECTORIES), libraries=list(EXTENSION_LIBRARIES), From 20b9f9432836b8959c034a01c51325d9c3b7c00a Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Fri, 7 Apr 2017 12:50:33 -0700 Subject: [PATCH 08/60] Update TSI with new non-blocking TSI interfaces. --- Makefile | 3 + src/core/tsi/fake_transport_security.c | 1 + src/core/tsi/ssl_transport_security.c | 1 + src/core/tsi/transport_security.c | 44 +++ src/core/tsi/transport_security.h | 22 ++ src/core/tsi/transport_security_adapter.c | 230 ++++++++++++++++ src/core/tsi/transport_security_adapter.h | 55 ++++ src/core/tsi/transport_security_interface.h | 279 ++++++++++++++++---- test/core/tsi/transport_security_test.c | 2 + 9 files changed, 585 insertions(+), 52 deletions(-) create mode 100644 src/core/tsi/transport_security_adapter.c create mode 100644 src/core/tsi/transport_security_adapter.h diff --git a/Makefile b/Makefile index 91a74f354e8..cbf38709911 100644 --- a/Makefile +++ b/Makefile @@ -2987,6 +2987,7 @@ LIBGRPC_SRC = \ src/core/tsi/fake_transport_security.c \ src/core/tsi/ssl_transport_security.c \ src/core/tsi/transport_security.c \ + src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \ src/core/ext/filters/client_channel/channel_connectivity.c \ @@ -3330,6 +3331,7 @@ LIBGRPC_CRONET_SRC = \ src/core/tsi/fake_transport_security.c \ src/core/tsi/ssl_transport_security.c \ src/core/tsi/transport_security.c \ + src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/client/chttp2_connector.c \ src/core/ext/filters/load_reporting/load_reporting.c \ src/core/ext/filters/load_reporting/load_reporting_filter.c \ @@ -19257,6 +19259,7 @@ src/core/plugin_registry/grpc_plugin_registry.c: $(OPENSSL_DEP) src/core/tsi/fake_transport_security.c: $(OPENSSL_DEP) src/core/tsi/ssl_transport_security.c: $(OPENSSL_DEP) src/core/tsi/transport_security.c: $(OPENSSL_DEP) +src/core/tsi/transport_security_adapter.c: $(OPENSSL_DEP) src/cpp/client/cronet_credentials.cc: $(OPENSSL_DEP) src/cpp/client/secure_credentials.cc: $(OPENSSL_DEP) src/cpp/common/auth_property_iterator.cc: $(OPENSSL_DEP) diff --git a/src/core/tsi/fake_transport_security.c b/src/core/tsi/fake_transport_security.c index 822fad51cb3..1836beefc42 100644 --- a/src/core/tsi/fake_transport_security.c +++ b/src/core/tsi/fake_transport_security.c @@ -499,6 +499,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { fake_handshaker_extract_peer, fake_handshaker_create_frame_protector, fake_handshaker_destroy, + NULL, }; tsi_handshaker *tsi_create_fake_handshaker(int is_client) { diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index a0325cc1835..c4dba002f8b 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -1049,6 +1049,7 @@ static const tsi_handshaker_vtable handshaker_vtable = { ssl_handshaker_extract_peer, ssl_handshaker_create_frame_protector, ssl_handshaker_destroy, + NULL, }; /* --- tsi_ssl_handshaker_factory common methods. --- */ diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index 67ebe1b1f30..a9cb6a107ca 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -73,6 +73,8 @@ const char *tsi_result_to_string(tsi_result result) { return "TSI_HANDSHAKE_IN_PROGRESS"; case TSI_OUT_OF_RESOURCES: return "TSI_OUT_OF_RESOURCES"; + case TSI_ASYNC: + return "TSI_ASYNC"; default: return "UNKNOWN"; } @@ -185,11 +187,53 @@ tsi_result tsi_handshaker_create_frame_protector( return result; } +tsi_result tsi_handshaker_next( + tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb cb, void *user_data) { + if (self == NULL) return TSI_INVALID_ARGUMENT; + if (self->handshaker_result_created) return TSI_FAILED_PRECONDITION; + return self->vtable->next(self, received_bytes, received_bytes_size, + bytes_to_send, bytes_to_send_size, + handshaker_result, cb, user_data); +} + void tsi_handshaker_destroy(tsi_handshaker *self) { if (self == NULL) return; self->vtable->destroy(self); } +/* --- tsi_handshaker_result implementation. --- */ + +tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, + tsi_peer *peer) { + if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT; + memset(peer, 0, sizeof(tsi_peer)); + return self->vtable->extract_peer(self, peer); +} + +tsi_result tsi_handshaker_result_create_frame_protector( + tsi_handshaker_result *self, size_t *max_protected_frame_size, + tsi_frame_protector **protector) { + if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT; + return self->vtable->create_frame_protector(self, max_protected_frame_size, + protector); +} + +tsi_result tsi_handshaker_result_get_unused_bytes( + tsi_handshaker_result *self, unsigned char **bytes, size_t *bytes_size) { + if (self == NULL || bytes == NULL || bytes_size == NULL) { + return TSI_INVALID_ARGUMENT; + } + return self->vtable->get_unused_bytes(self, bytes, bytes_size); +} + +void tsi_handshaker_result_destroy(tsi_handshaker_result *self) { + if (self == NULL) return; + self->vtable->destroy(self); +} + /* --- tsi_peer implementation. --- */ tsi_peer_property tsi_init_peer_property(void) { diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index 491fa1a8bd6..2e821108279 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -81,11 +81,33 @@ typedef struct { size_t *max_protected_frame_size, tsi_frame_protector **protector); void (*destroy)(tsi_handshaker *self); + tsi_result (*next)(tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, + tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb cb, void *user_data); } tsi_handshaker_vtable; struct tsi_handshaker { const tsi_handshaker_vtable *vtable; int frame_protector_created; + int handshaker_result_created; +}; + +/* Base for tsi_handshaker_result implementations. + See transport_security_interface.h for documentation. */ +typedef struct { + tsi_result (*extract_peer)(tsi_handshaker_result *self, tsi_peer *peer); + tsi_result (*create_frame_protector)(tsi_handshaker_result *self, + size_t *max_output_protected_frame_size, + tsi_frame_protector **protector); + tsi_result (*get_unused_bytes)(tsi_handshaker_result *self, + unsigned char **bytes, size_t *bytes_size); + void (*destroy)(tsi_handshaker_result *self); +} tsi_handshaker_result_vtable; + +struct tsi_handshaker_result { + const tsi_handshaker_result_vtable *vtable; }; /* Peer and property construction/destruction functions. */ diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c new file mode 100644 index 00000000000..7e0b8f574a0 --- /dev/null +++ b/src/core/tsi/transport_security_adapter.c @@ -0,0 +1,230 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/tsi/transport_security_adapter.h" + +#include + +#include +#include +#include "src/core/tsi/transport_security.h" + +#define TSI_ADAPTER_INITIAL_BUFFER_SIZE 256 + +/* --- tsi_adapter_handshaker_result implementation ---*/ + +typedef struct { + tsi_handshaker_result base; + tsi_handshaker *handshaker; + unsigned char *unused_bytes; + size_t unused_bytes_size; +} tsi_adapter_handshaker_result; + +static tsi_result tsi_adapter_result_extract_peer(tsi_handshaker_result *self, + tsi_peer *peer) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + return tsi_handshaker_extract_peer(impl->handshaker, peer); +} + +static tsi_result tsi_adapter_result_create_frame_protector( + tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_frame_protector **protector) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + return tsi_handshaker_create_frame_protector( + impl->handshaker, max_output_protected_frame_size, protector); +} + +static tsi_result tsi_adapter_result_get_unused_bytes( + tsi_handshaker_result *self, unsigned char **bytes, size_t *byte_size) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + *bytes = impl->unused_bytes; + *byte_size = impl->unused_bytes_size; + return TSI_OK; +} + +static void tsi_adapter_result_destroy(tsi_handshaker_result *self) { + tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + if (impl->unused_bytes != NULL) { + gpr_free(impl->unused_bytes); + } + gpr_free(self); +} + +static const tsi_handshaker_result_vtable result_vtable = { + tsi_adapter_result_extract_peer, tsi_adapter_result_create_frame_protector, + tsi_adapter_result_get_unused_bytes, tsi_adapter_result_destroy, +}; + +tsi_result tsi_adapter_create_handshaker_result( + tsi_handshaker *handshaker, const unsigned char *unused_bytes, + size_t unused_bytes_size, tsi_handshaker_result **handshaker_result) { + if (handshaker == NULL || (unused_bytes_size > 0 && unused_bytes == NULL)) { + return TSI_INVALID_ARGUMENT; + } + tsi_adapter_handshaker_result *impl = gpr_zalloc(sizeof(*impl)); + impl->base.vtable = &result_vtable; + impl->handshaker = handshaker; + impl->unused_bytes_size = unused_bytes_size; + if (unused_bytes_size > 0) { + impl->unused_bytes = gpr_malloc(unused_bytes_size); + memcpy(impl->unused_bytes, unused_bytes, unused_bytes_size); + } else { + impl->unused_bytes = NULL; + } + *handshaker_result = &impl->base; + return TSI_OK; +} + +/* --- tsi_adapter_handshaker implementation ---*/ + +typedef struct { + tsi_handshaker base; + tsi_handshaker *wrapped; + unsigned char *adapter_buffer; + size_t adapter_buffer_size; +} tsi_adapter_handshaker; + +tsi_result tsi_adapter_get_bytes_to_send_to_peer(tsi_handshaker *self, + unsigned char *bytes, + size_t *bytes_size) { + return tsi_handshaker_get_bytes_to_send_to_peer( + tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); +} + +tsi_result tsi_adapter_process_bytes_from_peer(tsi_handshaker *self, + const unsigned char *bytes, + size_t *bytes_size) { + return tsi_handshaker_process_bytes_from_peer( + tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); +} + +tsi_result tsi_adapter_get_result(tsi_handshaker *self) { + return tsi_handshaker_get_result(tsi_adapter_handshaker_get_wrapped(self)); +} + +tsi_result tsi_adapter_extract_peer(tsi_handshaker *self, tsi_peer *peer) { + return tsi_handshaker_extract_peer(tsi_adapter_handshaker_get_wrapped(self), + peer); +} + +tsi_result tsi_adapter_create_frame_protector(tsi_handshaker *self, + size_t *max_protected_frame_size, + tsi_frame_protector **protector) { + return tsi_handshaker_create_frame_protector( + tsi_adapter_handshaker_get_wrapped(self), max_protected_frame_size, + protector); +} + +void tsi_adapter_destroy(tsi_handshaker *self) { + tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self; + tsi_handshaker_destroy(impl->wrapped); + gpr_free(impl->adapter_buffer); + gpr_free(self); +} + +tsi_result tsi_adapter_next( + tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb cb, void *user_data) { + /* Input sanity check. */ + if ((received_bytes_size > 0 && received_bytes == NULL) || + bytes_to_send == NULL || bytes_to_send_size == NULL || + handshaker_result == NULL) { + return TSI_INVALID_ARGUMENT; + } + + /* If there are received bytes, process them first. */ + tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self; + tsi_result status = TSI_OK; + size_t bytes_consumed = received_bytes_size; + if (received_bytes_size > 0) { + status = tsi_handshaker_process_bytes_from_peer( + impl->wrapped, received_bytes, &bytes_consumed); + if (status != TSI_OK) return status; + } + + /* Get bytes to send to the peer, if available. */ + size_t offset = 0; + do { + size_t to_send_size = impl->adapter_buffer_size - offset; + status = tsi_handshaker_get_bytes_to_send_to_peer( + impl->wrapped, impl->adapter_buffer + offset, &to_send_size); + offset += to_send_size; + if (status == TSI_INCOMPLETE_DATA) { + impl->adapter_buffer_size *= 2; + impl->adapter_buffer = + gpr_realloc(impl->adapter_buffer, impl->adapter_buffer_size); + } + } while (status == TSI_INCOMPLETE_DATA); + if (status != TSI_OK) return status; + *bytes_to_send = impl->adapter_buffer; + *bytes_to_send_size = offset; + + /* If handshake completes, create tsi_handshaker_result. */ + if (!tsi_handshaker_is_in_progress(impl->wrapped)) { + size_t unused_bytes_size = received_bytes_size - bytes_consumed; + const unsigned char *unused_bytes = + unused_bytes_size == 0 ? NULL : received_bytes + bytes_consumed; + return tsi_adapter_create_handshaker_result( + impl->wrapped, unused_bytes, unused_bytes_size, handshaker_result); + } + *handshaker_result = NULL; + return TSI_OK; +} + +static const tsi_handshaker_vtable handshaker_vtable = { + tsi_adapter_get_bytes_to_send_to_peer, + tsi_adapter_process_bytes_from_peer, + tsi_adapter_get_result, + tsi_adapter_extract_peer, + tsi_adapter_create_frame_protector, + tsi_adapter_destroy, + tsi_adapter_next, +}; + +tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped) { + GPR_ASSERT(wrapped != NULL); + tsi_adapter_handshaker *impl = gpr_zalloc(sizeof(*impl)); + impl->base.vtable = &handshaker_vtable; + impl->wrapped = wrapped; + impl->adapter_buffer_size = TSI_ADAPTER_INITIAL_BUFFER_SIZE; + impl->adapter_buffer = gpr_malloc(impl->adapter_buffer_size); + return &impl->base; +} + +tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter) { + if (adapter == NULL) return NULL; + tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)adapter; + return impl->wrapped; +} diff --git a/src/core/tsi/transport_security_adapter.h b/src/core/tsi/transport_security_adapter.h new file mode 100644 index 00000000000..de5aaefd5f1 --- /dev/null +++ b/src/core/tsi/transport_security_adapter.h @@ -0,0 +1,55 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H +#define GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H + +#include "src/core/tsi/transport_security_interface.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Create a tsi handshaker that takes an implementation of old interface and + converts into an implementation of new interface. + Ownership of input tsi_handshaker is transferred to this new adapter. */ +tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped); + +/* Given a tsi adapter handshaker, return the original wrapped handshaker. */ +tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter); + +#ifdef __cplusplus +} +#endif + +#endif // GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index caed43eac44..be04810e24f 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -56,7 +56,8 @@ typedef enum { TSI_NOT_FOUND = 9, TSI_PROTOCOL_FAILURE = 10, TSI_HANDSHAKE_IN_PROGRESS = 11, - TSI_OUT_OF_RESOURCES = 12 + TSI_OUT_OF_RESOURCES = 12, + TSI_ASYNC = 13 } tsi_result; typedef enum { @@ -208,76 +209,203 @@ typedef struct { /* Destructs the tsi_peer object. */ void tsi_peer_destruct(tsi_peer *self); +/* --- tsi_handshaker_result object --- + + This object contains all necessary handshake results and data such as peer + info, negotiated keys, unused handshake bytes, when the handshake completes. + Implementations of this object must be thread compatible. */ + +typedef struct tsi_handshaker_result tsi_handshaker_result; + +/* This method extracts tsi peer. It returns TSI_OK assuming there is no fatal + error. + The caller is responsible for destructing the peer. */ +tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, + tsi_peer *peer); + +/* This method creates a tsi_frame_protector object. It returns TSI_OK assuming + there is no fatal error. + The caller is responsible for destroying the protector. */ +tsi_result tsi_handshaker_result_create_frame_protector( + tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + tsi_frame_protector **protector); + +/* This method returns the unused bytes from the handshake. It returns TSI_OK + assuming there is no fatal error. + The caller should not free the bytes. */ +tsi_result tsi_handshaker_result_get_unused_bytes(tsi_handshaker_result *self, + unsigned char **bytes, + size_t *byte_size); + +/* This method releases the tsi_handshaker_handshaker object. After this method + is called, no other method can be called on the object. */ +void tsi_handshaker_result_destroy(tsi_handshaker_result *self); + /* --- tsi_handshaker objects ---- Implementations of this object must be thread compatible. - A typical usage of this object would be: + ------------------------------------------------------------------------ + + A typical usage of the synchronous TSI handshaker would be: ------------------------------------------------------------------------ - tsi_result result = TSI_OK; + tsi_result status = TSI_OK; unsigned char buf[4096]; - size_t buf_offset; - size_t buf_size; + const size_t buf_size = 4906; + size_t bytes_received_size = 0; + unsigned char *bytes_to_send = NULL; + size_t bytes_to_send_size = 0; + tsi_handshaker_result *result = NULL; + while (1) { - // See if we need to send some bytes to the peer. - do { - size_t buf_size_to_send = sizeof(buf); - result = tsi_handshaker_get_bytes_to_send_to_peer(handshaker, buf, - &buf_size_to_send); - if (buf_size_to_send > 0) send_bytes_to_peer(buf, buf_size_to_send); - } while (result == TSI_INCOMPLETE_DATA); - if (result != TSI_OK) return result; - if (!tsi_handshaker_is_in_progress(handshaker)) break; - - do { - // Read bytes from the peer. - buf_size = sizeof(buf); - buf_offset = 0; - read_bytes_from_peer(buf, &buf_size); - if (buf_size == 0) break; - - // Process the bytes from the peer. We have to be careful as these bytes - // may contain non-handshake data (protected data). If this is the case, - // we will exit from the loop with buf_size > 0. - size_t consumed_by_handshaker = buf_size; - result = tsi_handshaker_process_bytes_from_peer( - handshaker, buf, &consumed_by_handshaker); - buf_size -= consumed_by_handshaker; - buf_offset += consumed_by_handshaker; - } while (result == TSI_INCOMPLETE_DATA); - - if (result != TSI_OK) return result; - if (!tsi_handshaker_is_in_progress(handshaker)) break; + status = tsi_handshaker_next( + handshaker, buf, bytes_received_size, + &bytes_to_send, &bytes_to_send_size, &result, NULL, NULL); + if (status == TSI_INCOMPLETE_DATA) { + // Need more data from the peer. + bytes_received_size = buf_size; + read_bytes_from_peer(buf, &bytes_received_size); + continue; + } + if (status != TSI_OK) return status; + if (bytes_to_send_size > 0) { + send_bytes_to_peer(bytes_to_send, bytes_to_send_size); + } + if (result != NULL) break; + bytes_received_size = buf_size; + read_bytes_from_peer(buf, &bytes_received_size); } // Check the Peer. tsi_peer peer; - do { - result = tsi_handshaker_extract_peer(handshaker, &peer); - if (result != TSI_OK) break; - result = check_peer(&peer); - } while (0); + status = tsi_handshaker_result_extract_peer(result, &peer); + if (status != TSI_OK) return status; + status = check_peer(&peer); tsi_peer_destruct(&peer); - if (result != TSI_OK) return result; + if (status != TSI_OK) return status; // Create the protector. tsi_frame_protector* protector = NULL; - result = tsi_handshaker_create_frame_protector(handshaker, NULL, - &protector); - if (result != TSI_OK) return result; + status = tsi_handshaker_result_create_frame_protector(result, NULL, + &protector); + if (status != TSI_OK) return status; // Do not forget to unprotect outstanding data if any. - if (buf_size > 0) { - result = tsi_frame_protector_unprotect(protector, buf + buf_offset, - buf_size, ..., ...); + unsigned char *unused_bytes = NULL; + size_t unused_bytes_size = 0; + status = tsi_handshaker_result_get_unused_bytes(result, &unused_bytes, + &unused_bytes_size); + if (status != TSI_OK) return status; + if (unused_bytes_size > 0) { + status = tsi_frame_protector_unprotect(protector, unused_bytes, + unused_bytes_size, ..., ...); .... } ... + ------------------------------------------------------------------------ + + A typical usage of the new TSI would be as follows, supporting both + synchronous and asynchrnous TSI handshaker implementations: + + ------------------------------------------------------------------------ + + typedef struct { + tsi_handshaker *handshaker; + tsi_handshaker_result *handshaker_result; + unsigned char *handshake_buffer; + size_t handshake_buffer_size; + ... + } security_handshaker; + + void do_handshake(security_handshaker *h, ...) { + // Start the handshake by the calling do_handshake_next. + do_handshake_next(h, NULL, 0); + ... + } + + // This method is a wrapper of the callback function to execute when + // tsi_handshaker_next finishes. It is passed to tsi_handshaker_next as + // the callback function. + void on_handshake_next_done_wrapper( + tsi_result status, void *user_data, const unsigned char *bytes_to_send, + size_t bytes_to_send_size, tsi_handshaker_result *result) { + security_handshaker *h = (security_handshaker *)user_data; + on_handshake_next_done(h, status, bytes_to_send, + bytes_to_send_size, result); + } + + // This method is the callback function when there are data received from + // the peer. This method will read bytes into the handshake buffer and call + // do_handshake_next. + void on_handshake_data_received_from_peer(void *user_data) { + security_handshaker *h = (security_handshaker *)user_data; + size_t bytes_received_size = h->handshake_buffer_size; + read_bytes_from_peer(h->handshake_buffer, &bytes_received_size); + do_handshake_next(h, h->handshake_buffer, bytes_received_size); + } + + // This method processes a step of handshake, calling tsi_handshaker_next. + void do_handshake_next(security_handshaker *h, + const unsigned char* bytes_received, + size_t bytes_received_size) { + tsi_result status = TSI_OK; + unsigned char *bytes_to_send = NULL; + size_t bytes_to_send_size = 0; + tsi_handshaker_result *result = NULL; + status = tsi_handshaker_next( + handshaker, bytes_received, bytes_received_size, &bytes_to_send, + &bytes_to_send_size, &result, on_handshake_next_done_wrapper, h); + // If TSI handshaker is asynchronous, on_handshake_next_done will be + // called during the execution of the callback function. + if (status == TSI_ASYNC) return; + on_handshake_next_done(h, status, bytes_to_send, + bytes_to_send_size, result); + } + + // This is the real function to execute after tsi_handshaker_next. + void on_handshake_next_done( + security_handshaker *h, tsi_result status, + const unsigned char *bytes_to_send, size_t bytes_to_send_size, + tsi_handshaker_result *result) { + if (status == TSI_INCOMPLETE_DATA) { + // Schedule an asynchronous read from the peer. If handshake data are + // received, on_handshake_data_received_from_peer will be called. + async_read_from_peer(..., ..., on_handshake_data_received_from_peer); + return; + } + if (status != TSI_OK) return; + + if (bytes_to_send_size > 0) { + send_bytes_to_peer(bytes_to_send, bytes_to_send_size); + } + + if (result != NULL) { + // Handshake completed. + h->result = result; + // Check the Peer. + tsi_peer peer; + status = tsi_handshaker_result_extract_peer(result, &peer); + if (status != TSI_OK) return status; + status = check_peer(&peer); + tsi_peer_destruct(&peer); + if (status != TSI_OK) return; + + // Create the protector. + tsi_frame_protector* protector = NULL; + status = tsi_handshaker_result_create_frame_protector(result, NULL, + &protector); + if (status != TSI_OK) return; + + // Do not forget to unprotect outstanding data if any. + .... + } + } ------------------------------------------------------------------------ */ typedef struct tsi_handshaker tsi_handshaker; -/* Gets bytes that need to be sent to the peer. +/* TO BE DEPRECATED SOON. + Gets bytes that need to be sent to the peer. - bytes is the buffer that will be written with the data to be sent to the peer. - bytes_size is an input/output parameter specifying the capacity of the @@ -292,7 +420,8 @@ tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, unsigned char *bytes, size_t *bytes_size); -/* Processes bytes received from the peer. +/* TO BE DEPRECATED SOON. + Processes bytes received from the peer. - bytes is the buffer containing the data. - bytes_size is an input/output parameter specifying the size of the data as input and the number of bytes consumed as output. @@ -305,24 +434,28 @@ tsi_result tsi_handshaker_process_bytes_from_peer(tsi_handshaker *self, const unsigned char *bytes, size_t *bytes_size); -/* Gets the result of the handshaker. +/* TO BE DEPRECATED SOON. + Gets the result of the handshaker. Returns TSI_OK if the hanshake completed successfully and there has been no errors. Returns TSI_HANDSHAKE_IN_PROGRESS if the handshaker is not done yet but no error has been encountered so far. Otherwise the handshaker failed with the returned error. */ tsi_result tsi_handshaker_get_result(tsi_handshaker *self); -/* Returns 1 if the handshake is in progress, 0 otherwise. */ +/* TO BE DEPRECATED SOON. + Returns 1 if the handshake is in progress, 0 otherwise. */ #define tsi_handshaker_is_in_progress(h) \ (tsi_handshaker_get_result((h)) == TSI_HANDSHAKE_IN_PROGRESS) -/* This method may return TSI_FAILED_PRECONDITION if +/* TO BE DEPRECATED SOON. + This method may return TSI_FAILED_PRECONDITION if tsi_handshaker_is_in_progress returns 1, it returns TSI_OK otherwise assuming the handshaker is not in a fatal error state. The caller is responsible for destructing the peer. */ tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer); -/* This method creates a tsi_frame_protector object after the handshake phase +/* TO BE DEPRECATED SOON. + This method creates a tsi_frame_protector object after the handshake phase is done. After this method has been called successfully, the only method that can be called on this object is Destroy. - max_output_protected_frame_size is an input/output parameter specifying the @@ -342,10 +475,52 @@ tsi_result tsi_handshaker_create_frame_protector( tsi_handshaker *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector); +/* Callback function definition for tsi_handshaker_next. + - status indicates the status of the next operation. + - user_data is the argument to callback function passed from the caller. + - bytes_to_send is the data buffer to be sent to the peer. + - bytes_to_send_size is the size of data buffer to be sent to the peer. + - handshaker_result is the result of handshake when the handshake completes, + is NULL otherwise. */ +typedef void (*tsi_handshaker_on_next_done_cb)( + tsi_result status, void *user_data, const unsigned char *bytes_to_send, + size_t bytes_to_send_size, tsi_handshaker_result *handshaker_result); + +/* Conduct a next step of the handshake. + - received_bytes is the buffer containing the data received from the peer. + - received_bytes_size is the size of the data received from the peer. + - bytes_to_send is the data buffer to be sent to the peer. + - bytes_to_send_size is the size of data buffer to be sent to the peer. + - handshaker_result is the result of handshake if the handshake completes. + - cb is the callback function defined above. It can be NULL for synchronous + TSI handshaker implementation. + - user_data is the argument to callback function passed from the caller. + This method returns TSI_ASYNC if the TSI handshaker implementation is + asynchronous. It returns TSI_OK if the handshake completes or if there are + data to send to the peer, otherwise returns TSI_INCOMPLETE_DATA which + indicates that this method needs to be called again with more data from the + peer. In case of a fatal error in the handshake, another specific error code + is returned. + The caller is responsible for destroying the handshaker_result. However, the + caller should not free bytes_to_send, as the buffer is owned by the + tsi_handshaker object. */ +tsi_result tsi_handshaker_next( + tsi_handshaker *self, const unsigned char *received_bytes, + size_t received_bytes_size, unsigned char **bytes_to_send, + size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, + tsi_handshaker_on_next_done_cb cb, void *user_data); + /* This method releases the tsi_handshaker object. After this method is called, no other method can be called on the object. */ void tsi_handshaker_destroy(tsi_handshaker *self); +/* This method initializes the necessary shared objects used for tsi + implementation. */ +void tsi_init(); + +/* This method destroys the shared objects created by tsi_init. */ +void tsi_destroy(); + #ifdef __cplusplus } #endif diff --git a/test/core/tsi/transport_security_test.c b/test/core/tsi/transport_security_test.c index ee4a37c3141..4214407823a 100644 --- a/test/core/tsi/transport_security_test.c +++ b/test/core/tsi/transport_security_test.c @@ -376,6 +376,8 @@ static void test_handshaker_invalid_args(void) { TSI_INVALID_ARGUMENT); GPR_ASSERT(tsi_handshaker_get_bytes_to_send_to_peer(NULL, NULL, NULL) == TSI_INVALID_ARGUMENT); + GPR_ASSERT(tsi_handshaker_next(NULL, NULL, 0, NULL, NULL, NULL, NULL, NULL) == + TSI_INVALID_ARGUMENT); } static void test_handshaker_invalid_state(void) { From f674f8e825b4ded4985e36205a5ff0bfd855872a Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Sun, 9 Apr 2017 21:48:22 -0700 Subject: [PATCH 09/60] *san needs clang++ to link --- CMakeLists.txt | 4 ++-- Makefile | 16 ++++++++-------- build.yaml | 12 ++++++------ tools/doxygen/Doxyfile.c++.internal | 2 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj | 2 +- vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters | 2 +- .../grpc++_unsecure/grpc++_unsecure.vcxproj | 2 +- .../grpc++_unsecure.vcxproj.filters | 2 +- 8 files changed, 21 insertions(+), 21 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index db0c41ff92b..682b5ab82d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2320,7 +2320,7 @@ add_library(grpc++ src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c @@ -3348,7 +3348,7 @@ add_library(grpc++_unsecure src/core/lib/surface/completion_queue.c src/core/lib/surface/completion_queue_factory.c src/core/lib/surface/event_string.c - src/core/lib/surface/lame_client.c + src/core/lib/surface/lame_client.cc src/core/lib/surface/metadata_array.c src/core/lib/surface/server.c src/core/lib/surface/validate_metadata.c diff --git a/Makefile b/Makefile index e6726cb54f7..a55e984efd3 100644 --- a/Makefile +++ b/Makefile @@ -99,7 +99,7 @@ VALID_CONFIG_asan-trace-cmp = 1 REQUIRE_CUSTOM_LIBRARIES_asan-trace-cmp = 1 CC_asan-trace-cmp = clang CXX_asan-trace-cmp = clang++ -LD_asan-trace-cmp = clang +LD_asan-trace-cmp = clang++ LDXX_asan-trace-cmp = clang++ CPPFLAGS_asan-trace-cmp = -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-trace-cmp = -fsanitize=address @@ -116,7 +116,7 @@ VALID_CONFIG_asan = 1 REQUIRE_CUSTOM_LIBRARIES_asan = 1 CC_asan = clang CXX_asan = clang++ -LD_asan = clang +LD_asan = clang++ LDXX_asan = clang++ CPPFLAGS_asan = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan = -fsanitize=address @@ -125,7 +125,7 @@ VALID_CONFIG_msan = 1 REQUIRE_CUSTOM_LIBRARIES_msan = 1 CC_msan = clang CXX_msan = clang++ -LD_msan = clang +LD_msan = clang++ LDXX_msan = clang++ CPPFLAGS_msan = -O0 -fsanitize-coverage=edge -fsanitize=memory -fsanitize-memory-track-origins -fno-omit-frame-pointer -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_msan = -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) @@ -152,7 +152,7 @@ VALID_CONFIG_asan-noleaks = 1 REQUIRE_CUSTOM_LIBRARIES_asan-noleaks = 1 CC_asan-noleaks = clang CXX_asan-noleaks = clang++ -LD_asan-noleaks = clang +LD_asan-noleaks = clang++ LDXX_asan-noleaks = clang++ CPPFLAGS_asan-noleaks = -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_asan-noleaks = -fsanitize=address @@ -161,7 +161,7 @@ VALID_CONFIG_ubsan = 1 REQUIRE_CUSTOM_LIBRARIES_ubsan = 1 CC_ubsan = clang CXX_ubsan = clang++ -LD_ubsan = clang +LD_ubsan = clang++ LDXX_ubsan = clang++ CPPFLAGS_ubsan = -O0 -fsanitize-coverage=edge -fsanitize=undefined -fno-omit-frame-pointer -Wno-unused-command-line-argument -Wvarargs LDFLAGS_ubsan = -fsanitize=undefined,unsigned-integer-overflow @@ -171,7 +171,7 @@ VALID_CONFIG_tsan = 1 REQUIRE_CUSTOM_LIBRARIES_tsan = 1 CC_tsan = clang CXX_tsan = clang++ -LD_tsan = clang +LD_tsan = clang++ LDXX_tsan = clang++ CPPFLAGS_tsan = -O0 -fsanitize=thread -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS LDFLAGS_tsan = -fsanitize=thread @@ -4177,7 +4177,7 @@ LIBGRPC++_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ @@ -5205,7 +5205,7 @@ LIBGRPC++_UNSECURE_SRC = \ src/core/lib/surface/completion_queue.c \ src/core/lib/surface/completion_queue_factory.c \ src/core/lib/surface/event_string.c \ - src/core/lib/surface/lame_client.c \ + src/core/lib/surface/lame_client.cc \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ src/core/lib/surface/validate_metadata.c \ diff --git a/build.yaml b/build.yaml index f2ea2b12045..f80c259606b 100644 --- a/build.yaml +++ b/build.yaml @@ -4288,7 +4288,7 @@ configs: CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ - LD: clang + LD: clang++ LDFLAGS: -fsanitize=address LDXX: clang++ compile_the_world: true @@ -4300,7 +4300,7 @@ configs: CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ - LD: clang + LD: clang++ LDFLAGS: -fsanitize=address LDXX: clang++ compile_the_world: true @@ -4311,7 +4311,7 @@ configs: CPPFLAGS: -O0 -fsanitize-coverage=edge -fsanitize-coverage=trace-cmp -fsanitize=address -fno-omit-frame-pointer -Wno-unused-command-line-argument -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ - LD: clang + LD: clang++ LDFLAGS: -fsanitize=address LDXX: clang++ compile_the_world: true @@ -4355,7 +4355,7 @@ configs: -Wno-unused-command-line-argument -fPIE -pie -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ DEFINES: NDEBUG - LD: clang + LD: clang++ LDFLAGS: -fsanitize=memory -DGTEST_HAS_TR1_TUPLE=0 -DGTEST_USE_OWN_TR1_TUPLE=1 -fPIE -pie $(if $(JENKINS_BUILD),-Wl$(comma)-Ttext-segment=0x7e0000000000,) LDXX: clang++ @@ -4376,7 +4376,7 @@ configs: -DGPR_NO_DIRECT_SYSCALLS CXX: clang++ DEFINES: GRPC_TSAN - LD: clang + LD: clang++ LDFLAGS: -fsanitize=thread LDXX: clang++ compile_the_world: true @@ -4388,7 +4388,7 @@ configs: -Wno-unused-command-line-argument -Wvarargs CXX: clang++ DEFINES: NDEBUG - LD: clang + LD: clang++ LDFLAGS: -fsanitize=undefined,unsigned-integer-overflow LDXX: clang++ compile_the_world: true diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 209d5445dbc..a4750f592a0 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1099,7 +1099,7 @@ src/core/lib/surface/completion_queue_factory.h \ src/core/lib/surface/event_string.c \ src/core/lib/surface/event_string.h \ src/core/lib/surface/init.h \ -src/core/lib/surface/lame_client.c \ +src/core/lib/surface/lame_client.cc \ src/core/lib/surface/lame_client.h \ src/core/lib/surface/metadata_array.c \ src/core/lib/surface/server.c \ diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj b/vsprojects/vcxproj/grpc++/grpc++.vcxproj index caa22a019db..e364bd4b0e6 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj @@ -787,7 +787,7 @@ - + diff --git a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters index 6fc1c969309..ed104a93a37 100644 --- a/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++/grpc++.vcxproj.filters @@ -442,7 +442,7 @@ src\core\lib\surface - + src\core\lib\surface diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj index 674818182e8..fdebb811406 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj @@ -771,7 +771,7 @@ - + diff --git a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters index 2b9a5b13c1a..6ffde5c508d 100644 --- a/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_unsecure/grpc++_unsecure.vcxproj.filters @@ -427,7 +427,7 @@ src\core\lib\surface - + src\core\lib\surface From f7796e0b2f482b98586d9bf831eb356b11025102 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Mon, 10 Apr 2017 11:07:49 -0700 Subject: [PATCH 10/60] Get tests to pass for c++ --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 88dc54f8d0d..6ea79fc20a7 100644 --- a/setup.py +++ b/setup.py @@ -172,7 +172,7 @@ LDFLAGS = tuple(EXTRA_LINK_ARGS) CFLAGS = tuple(EXTRA_COMPILE_ARGS) if "linux" in sys.platform or "darwin" in sys.platform: pymodinit_type = 'PyObject*' if PY3 else 'void' - pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type) + pymodinit = 'extern "C" __attribute__((visibility ("default"))) {}'.format(pymodinit_type) DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),) # By default, Python3 distutils enforces compatibility of From 7400d1ce7548328d0107cea3ebe4dd0488bfe642 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Fri, 7 Apr 2017 10:17:40 -0700 Subject: [PATCH 11/60] Ignore a couple of errors in the Node express benchmark --- .../performance/benchmark_client_express.js | 25 +++++++++++++------ .../performance/benchmark_server_express.js | 4 +-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/node/performance/benchmark_client_express.js b/src/node/performance/benchmark_client_express.js index 675eb5f2884..e749956599d 100644 --- a/src/node/performance/benchmark_client_express.js +++ b/src/node/performance/benchmark_client_express.js @@ -93,7 +93,7 @@ function BenchmarkClient(server_targets, channels, histogram_params, for (var i = 0; i < channels; i++) { var host_port; - host_port = server_targets[i % server_targets.length].split(':') + host_port = server_targets[i % server_targets.length].split(':'); var new_options = _.assign({hostname: host_port[0], port: +host_port[1]}, options); new_options.agent = new protocol.Agent(new_options); this.client_options[i] = new_options; @@ -149,6 +149,17 @@ BenchmarkClient.prototype.startClosedLoop = function( if (self.running) { self.pending_calls++; var start_time = process.hrtime(); + function finishCall(success) { + if (success) { + var time_diff = process.hrtime(start_time); + self.histogram.add(timeDiffToNanos(time_diff)); + } + makeCall(client_options); + self.pending_calls--; + if ((!self.running) && self.pending_calls == 0) { + self.emit('finished'); + } + } var req = self.request(client_options, function(res) { var res_data = ''; res.on('data', function(data) { @@ -156,18 +167,16 @@ BenchmarkClient.prototype.startClosedLoop = function( }); res.on('end', function() { JSON.parse(res_data); - var time_diff = process.hrtime(start_time); - self.histogram.add(timeDiffToNanos(time_diff)); - makeCall(client_options); - self.pending_calls--; - if ((!self.running) && self.pending_calls == 0) { - self.emit('finished'); - } + finishCall(true); }); }); req.write(JSON.stringify(argument)); req.end(); req.on('error', function(error) { + if (error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT') { + finishCall(false); + return; + } self.emit('error', new Error('Client error: ' + error.message)); self.running = false; }); diff --git a/src/node/performance/benchmark_server_express.js b/src/node/performance/benchmark_server_express.js index 065bcf660b6..4b695eb467d 100644 --- a/src/node/performance/benchmark_server_express.js +++ b/src/node/performance/benchmark_server_express.js @@ -46,7 +46,7 @@ var EventEmitter = require('events'); var util = require('util'); var express = require('express'); -var bodyParser = require('body-parser') +var bodyParser = require('body-parser'); function unaryCall(req, res) { var reqObj = req.body; @@ -56,7 +56,7 @@ function unaryCall(req, res) { function BenchmarkServer(host, port, tls, generic, response_size) { var app = express(); - app.use(bodyParser.json()) + app.use(bodyParser.json()); app.put('/serviceProto.BenchmarkService.service/unaryCall', unaryCall); this.input_host = host; this.input_port = port; From 9cb19b8e2429389dc68045aa7f61384f4b03660f Mon Sep 17 00:00:00 2001 From: yang-g Date: Fri, 7 Apr 2017 14:17:41 -0700 Subject: [PATCH 12/60] support C++ error_details --- include/grpc++/impl/codegen/call.h | 55 +++++++++++++++------- include/grpc++/impl/codegen/status.h | 22 ++++++--- src/proto/grpc/testing/echo_messages.proto | 9 ++++ test/cpp/end2end/end2end_test.cc | 33 +++++++++++++ test/cpp/end2end/test_service_impl.cc | 5 ++ 5 files changed, 100 insertions(+), 24 deletions(-) diff --git a/include/grpc++/impl/codegen/call.h b/include/grpc++/impl/codegen/call.h index be6857c4829..dd63c21ff17 100644 --- a/include/grpc++/impl/codegen/call.h +++ b/include/grpc++/impl/codegen/call.h @@ -63,21 +63,31 @@ class CallHook; class CompletionQueue; extern CoreCodegenInterface* g_core_codegen_interface; +const char kBinaryErrorDetailsKey[] = "grpc-status-details-bin"; + // TODO(yangg) if the map is changed before we send, the pointers will be a // mess. Make sure it does not happen. inline grpc_metadata* FillMetadataArray( - const std::multimap& metadata) { - if (metadata.empty()) { + const std::multimap& metadata, + size_t* metadata_count, const grpc::string& optional_error_details) { + *metadata_count = metadata.size() + (optional_error_details.empty() ? 0 : 1); + if (*metadata_count == 0) { return nullptr; } grpc_metadata* metadata_array = (grpc_metadata*)(g_core_codegen_interface->gpr_malloc( - metadata.size() * sizeof(grpc_metadata))); + (*metadata_count) * sizeof(grpc_metadata))); size_t i = 0; for (auto iter = metadata.cbegin(); iter != metadata.cend(); ++iter, ++i) { metadata_array[i].key = SliceReferencingString(iter->first); metadata_array[i].value = SliceReferencingString(iter->second); } + if (!optional_error_details.empty()) { + metadata_array[i].key = + g_core_codegen_interface->grpc_slice_from_static_buffer( + kBinaryErrorDetailsKey, sizeof(kBinaryErrorDetailsKey) - 1); + metadata_array[i].value = SliceReferencingString(optional_error_details); + } return metadata_array; } @@ -216,8 +226,8 @@ class CallOpSendInitialMetadata { maybe_compression_level_.is_set = false; send_ = true; flags_ = flags; - initial_metadata_count_ = metadata.size(); - initial_metadata_ = FillMetadataArray(metadata); + initial_metadata_ = + FillMetadataArray(metadata, &initial_metadata_count_, ""); } void set_compression_level(grpc_compression_level level) { @@ -454,11 +464,12 @@ class CallOpServerSendStatus { void ServerSendStatus( const std::multimap& trailing_metadata, const Status& status) { - trailing_metadata_count_ = trailing_metadata.size(); - trailing_metadata_ = FillMetadataArray(trailing_metadata); + send_error_details_ = status.error_details(); + trailing_metadata_ = FillMetadataArray( + trailing_metadata, &trailing_metadata_count_, send_error_details_); send_status_available_ = true; send_status_code_ = static_cast(GetCanonicalCode(status)); - send_status_details_ = status.error_message(); + send_error_message_ = status.error_message(); } protected: @@ -470,9 +481,9 @@ class CallOpServerSendStatus { trailing_metadata_count_; op->data.send_status_from_server.trailing_metadata = trailing_metadata_; op->data.send_status_from_server.status = send_status_code_; - status_details_slice_ = SliceReferencingString(send_status_details_); + error_message_slice_ = SliceReferencingString(send_error_message_); op->data.send_status_from_server.status_details = - send_status_details_.empty() ? nullptr : &status_details_slice_; + send_error_message_.empty() ? nullptr : &error_message_slice_; op->flags = 0; op->reserved = NULL; } @@ -486,10 +497,11 @@ class CallOpServerSendStatus { private: bool send_status_available_; grpc_status_code send_status_code_; - grpc::string send_status_details_; + grpc::string send_error_details_; + grpc::string send_error_message_; size_t trailing_metadata_count_; grpc_metadata* trailing_metadata_; - grpc_slice status_details_slice_; + grpc_slice error_message_slice_; }; class CallOpRecvInitialMetadata { @@ -528,7 +540,7 @@ class CallOpClientRecvStatus { void ClientRecvStatus(ClientContext* context, Status* status) { metadata_map_ = &context->trailing_metadata_; recv_status_ = status; - status_details_ = g_core_codegen_interface->grpc_empty_slice(); + error_message_ = g_core_codegen_interface->grpc_empty_slice(); } protected: @@ -538,7 +550,7 @@ class CallOpClientRecvStatus { op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; op->data.recv_status_on_client.trailing_metadata = metadata_map_->arr(); op->data.recv_status_on_client.status = &status_code_; - op->data.recv_status_on_client.status_details = &status_details_; + op->data.recv_status_on_client.status_details = &error_message_; op->flags = 0; op->reserved = NULL; } @@ -546,10 +558,17 @@ class CallOpClientRecvStatus { void FinishOp(bool* status) { if (recv_status_ == nullptr) return; metadata_map_->FillMap(); + grpc::string binary_error_details; + auto iter = metadata_map_->map()->find(kBinaryErrorDetailsKey); + if (iter != metadata_map_->map()->end()) { + binary_error_details = + grpc::string(iter->second.begin(), iter->second.length()); + } *recv_status_ = Status(static_cast(status_code_), - grpc::string(GRPC_SLICE_START_PTR(status_details_), - GRPC_SLICE_END_PTR(status_details_))); - g_core_codegen_interface->grpc_slice_unref(status_details_); + grpc::string(GRPC_SLICE_START_PTR(error_message_), + GRPC_SLICE_END_PTR(error_message_)), + binary_error_details); + g_core_codegen_interface->grpc_slice_unref(error_message_); recv_status_ = nullptr; } @@ -557,7 +576,7 @@ class CallOpClientRecvStatus { MetadataMap* metadata_map_; Status* recv_status_; grpc_status_code status_code_; - grpc_slice status_details_; + grpc_slice error_message_; }; /// An abstract collection of CallOpSet's, to be used whenever diff --git a/include/grpc++/impl/codegen/status.h b/include/grpc++/impl/codegen/status.h index a509d311d42..5cce3c1672e 100644 --- a/include/grpc++/impl/codegen/status.h +++ b/include/grpc++/impl/codegen/status.h @@ -47,10 +47,16 @@ class Status { /// Construct an OK instance. Status() : code_(StatusCode::OK) {} - /// Construct an instance with associated \a code and \a details (also - // referred to as "error_message"). - Status(StatusCode code, const grpc::string& details) - : code_(code), details_(details) {} + /// Construct an instance with associated \a code and \a error_message + Status(StatusCode code, const grpc::string& error_message) + : code_(code), error_message_(error_message) {} + + /// Construct an instance with \a code, \a error_message and \a error_details + Status(StatusCode code, const grpc::string& error_message, + const grpc::string error_details) + : code_(code), + error_message_(error_message), + binary_error_details_(error_details) {} // Pre-defined special status objects. /// An OK pre-defined instance. @@ -61,14 +67,18 @@ class Status { /// Return the instance's error code. StatusCode error_code() const { return code_; } /// Return the instance's error message. - grpc::string error_message() const { return details_; } + grpc::string error_message() const { return error_message_; } + /// Return the (binary) error details. + // Usually it contains a serialized google.rpc.Status proto. + grpc::string error_details() const { return binary_error_details_; } /// Is the status OK? bool ok() const { return code_ == StatusCode::OK; } private: StatusCode code_; - grpc::string details_; + grpc::string error_message_; + grpc::string binary_error_details_; }; } // namespace grpc diff --git a/src/proto/grpc/testing/echo_messages.proto b/src/proto/grpc/testing/echo_messages.proto index efb6f4d4935..b82e80d8e34 100644 --- a/src/proto/grpc/testing/echo_messages.proto +++ b/src/proto/grpc/testing/echo_messages.proto @@ -38,6 +38,13 @@ message DebugInfo { string detail = 2; } +// Error status client expects to see. +message ErrorStatus { + int32 code = 1; + string error_message = 2; + string binary_error_details = 3; +} + message RequestParams { bool echo_deadline = 1; int32 client_cancel_after_us = 2; @@ -51,6 +58,8 @@ message RequestParams { string expected_transport_security_type = 10; DebugInfo debug_info = 11; bool server_die = 12; // Server should not see a request with this set. + string binary_error_details = 13; + ErrorStatus expected_error = 14; } message EchoRequest { diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index d3a83b188f9..df71777e4bd 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -1130,6 +1130,39 @@ TEST_P(End2endTest, BinaryTrailerTest) { EXPECT_TRUE(returned_info.ParseFromString(ToString(iter->second))); } +TEST_P(End2endTest, ExpectErrorTest) { + ResetStub(); + + std::vector expected_status; + expected_status.emplace_back(); + expected_status.back().set_code(13); // INTERNAL + expected_status.back().set_error_message("text error message"); + expected_status.back().set_binary_error_details("text error details"); + expected_status.emplace_back(); + expected_status.back().set_code(13); // INTERNAL + expected_status.back().set_error_message("text error message"); + expected_status.back().set_binary_error_details( + "\x0\x1\x2\x3\x4\x5\x6\x8\x9\xA\xB"); + + for (auto iter = expected_status.begin(); iter != expected_status.end(); + ++iter) { + EchoRequest request; + EchoResponse response; + ClientContext context; + request.set_message("Hello"); + auto* error = request.mutable_param()->mutable_expected_error(); + error->set_code(iter->code()); + error->set_error_message(iter->error_message()); + error->set_binary_error_details(iter->binary_error_details()); + + Status s = stub_->Echo(&context, request, &response); + EXPECT_FALSE(s.ok()); + EXPECT_EQ(iter->code(), s.error_code()); + EXPECT_EQ(iter->error_message(), s.error_message()); + EXPECT_EQ(iter->binary_error_details(), s.error_details()); + } +} + ////////////////////////////////////////////////////////////////////////// // Test with and without a proxy. class ProxyEnd2endTest : public End2endTest { diff --git a/test/cpp/end2end/test_service_impl.cc b/test/cpp/end2end/test_service_impl.cc index 11729c425cd..b473dd1f52e 100644 --- a/test/cpp/end2end/test_service_impl.cc +++ b/test/cpp/end2end/test_service_impl.cc @@ -92,6 +92,11 @@ Status TestServiceImpl::Echo(ServerContext* context, const EchoRequest* request, gpr_log(GPR_ERROR, "The request should not reach application handler."); GPR_ASSERT(0); } + if (request->has_param() && request->param().has_expected_error()) { + const auto& error = request->param().expected_error(); + return Status(static_cast(error.code()), error.error_message(), + error.binary_error_details()); + } int server_try_cancel = GetIntValueFromMetadata( kServerTryCancelRequest, context->client_metadata(), DO_NOT_CANCEL); if (server_try_cancel > DO_NOT_CANCEL) { From 1b6097cfac5ebaeac8ec5d91478812fd4bf70add Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 13:28:45 -0700 Subject: [PATCH 13/60] Pass in -fno-exceptions because we dont need them --- Makefile | 2 ++ build.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index a55e984efd3..26a1cc5eeeb 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,7 @@ CC_opt = $(DEFAULT_CC) CXX_opt = $(DEFAULT_CXX) LD_opt = $(DEFAULT_CC) LDXX_opt = $(DEFAULT_CXX) +CXXFLAGS_opt = -fno-exceptions CPPFLAGS_opt = -O2 DEFINES_opt = NDEBUG @@ -109,6 +110,7 @@ CC_dbg = $(DEFAULT_CC) CXX_dbg = $(DEFAULT_CXX) LD_dbg = $(DEFAULT_CC) LDXX_dbg = $(DEFAULT_CXX) +CXXFLAGS_dbg = -fno-exceptions CPPFLAGS_dbg = -O0 DEFINES_dbg = _DEBUG DEBUG diff --git a/build.yaml b/build.yaml index f80c259606b..e26fa16968a 100644 --- a/build.yaml +++ b/build.yaml @@ -4326,6 +4326,7 @@ configs: DEFINES: NDEBUG dbg: CPPFLAGS: -O0 + CXXFLAGS: -fno-exceptions DEFINES: _DEBUG DEBUG gcov: CC: gcc @@ -4366,6 +4367,7 @@ configs: LDFLAGS: -rdynamic opt: CPPFLAGS: -O2 + CXXFLAGS: -fno-exceptions DEFINES: NDEBUG stapprof: CPPFLAGS: -O2 -DGRPC_STAP_PROFILER From f85aabf6597a8edc0a35c068f3506ca04e68d14d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 14:06:15 -0700 Subject: [PATCH 14/60] Make ruby build debuggable --- src/ruby/ext/grpc/extconf.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 70679338321..4f5d77012b1 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -84,7 +84,7 @@ unless windows puts 'Building internal gRPC into ' + grpc_lib_dir nproc = 4 nproc = Etc.nprocessors * 2 if Etc.respond_to? :nprocessors - system("make -j#{nproc} -C #{grpc_root} #{grpc_lib_dir}/libgrpc.a CONFIG=#{grpc_config}") + system("make -j#{nproc} -C #{grpc_root} #{grpc_lib_dir}/libgrpc.a CONFIG=#{grpc_config} Q=") exit 1 unless $? == 0 end From 157c5696342c91bbeb2dbd7088f814864c752bc2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 14:27:57 -0700 Subject: [PATCH 15/60] fix c# --- tools/run_tests/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index 0b4f26ca440..a58a7e2f201 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -757,7 +757,7 @@ class CSharpLanguage(object): self._make_options = ['EMBED_OPENSSL=true'] if self.args.compiler != 'coreclr': # On Mac, official distribution of mono is 32bit. - self._make_options += ['CFLAGS=-m32', 'LDFLAGS=-m32'] + self._make_options += ['CFLAGS=-m32', 'CXXFLAGS=-m32', 'LDFLAGS=-m32'] else: self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] From e69881de7fd5a2f09ad92986c48e5fa2a013bb34 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Mon, 10 Apr 2017 14:29:43 -0700 Subject: [PATCH 16/60] Revise based on Mark's comments. --- BUILD | 2 + CMakeLists.txt | 2 + binding.gyp | 1 + build.yaml | 2 + config.m4 | 1 + gRPC-Core.podspec | 3 ++ grpc.gemspec | 2 + package.xml | 2 + src/core/tsi/transport_security.c | 9 ++-- src/core/tsi/transport_security.h | 12 +++-- src/core/tsi/transport_security_adapter. | 0 src/core/tsi/transport_security_adapter.c | 49 ++++++++++--------- src/core/tsi/transport_security_interface.h | 44 +++++++---------- src/python/grpcio/grpc_core_dependencies.py | 1 + tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 3 ++ vsprojects/vcxproj/grpc/grpc.vcxproj | 3 ++ vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 +++ 18 files changed, 86 insertions(+), 58 deletions(-) create mode 100644 src/core/tsi/transport_security_adapter. diff --git a/BUILD b/BUILD index 1991c867432..70ce0ecb6bd 100644 --- a/BUILD +++ b/BUILD @@ -1149,12 +1149,14 @@ grpc_cc_library( "src/core/tsi/fake_transport_security.c", "src/core/tsi/ssl_transport_security.c", "src/core/tsi/transport_security.c", + "src/core/tsi/transport_security_adapter.c", ], hdrs = [ "src/core/tsi/fake_transport_security.h", "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_adapter.h", "src/core/tsi/transport_security_interface.h", ], external_deps = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index f8b88645cb1..3b11066625c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1080,6 +1080,7 @@ add_library(grpc src/core/tsi/fake_transport_security.c src/core/tsi/ssl_transport_security.c src/core/tsi/transport_security.c + src/core/tsi/transport_security_adapter.c src/core/ext/transport/chttp2/server/chttp2_server.c src/core/ext/transport/chttp2/client/secure/secure_channel_create.c src/core/ext/filters/client_channel/channel_connectivity.c @@ -1425,6 +1426,7 @@ add_library(grpc_cronet src/core/tsi/fake_transport_security.c src/core/tsi/ssl_transport_security.c src/core/tsi/transport_security.c + src/core/tsi/transport_security_adapter.c src/core/ext/transport/chttp2/client/chttp2_connector.c src/core/ext/filters/load_reporting/load_reporting.c src/core/ext/filters/load_reporting/load_reporting_filter.c diff --git a/binding.gyp b/binding.gyp index 3444fa53dba..b64ff58f5c5 100644 --- a/binding.gyp +++ b/binding.gyp @@ -822,6 +822,7 @@ 'src/core/tsi/fake_transport_security.c', 'src/core/tsi/ssl_transport_security.c', 'src/core/tsi/transport_security.c', + 'src/core/tsi/transport_security_adapter.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', 'src/core/ext/filters/client_channel/channel_connectivity.c', diff --git a/build.yaml b/build.yaml index 6fbc8d91666..e5bec63ed0a 100644 --- a/build.yaml +++ b/build.yaml @@ -783,11 +783,13 @@ filegroups: - src/core/tsi/ssl_transport_security.h - src/core/tsi/ssl_types.h - src/core/tsi/transport_security.h + - src/core/tsi/transport_security_adapter.h - src/core/tsi/transport_security_interface.h src: - src/core/tsi/fake_transport_security.c - src/core/tsi/ssl_transport_security.c - src/core/tsi/transport_security.c + - src/core/tsi/transport_security_adapter.c deps: - gpr secure: true diff --git a/config.m4 b/config.m4 index 28467810313..1be815e3f34 100644 --- a/config.m4 +++ b/config.m4 @@ -256,6 +256,7 @@ if test "$PHP_GRPC" != "no"; then src/core/tsi/fake_transport_security.c \ src/core/tsi/ssl_transport_security.c \ src/core/tsi/transport_security.c \ + src/core/tsi/transport_security_adapter.c \ src/core/ext/transport/chttp2/server/chttp2_server.c \ src/core/ext/transport/chttp2/client/secure/secure_channel_create.c \ src/core/ext/filters/client_channel/channel_connectivity.c \ diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 83c19afc243..78d2f157789 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -407,6 +407,7 @@ Pod::Spec.new do |s| 'src/core/tsi/ssl_transport_security.h', 'src/core/tsi/ssl_types.h', 'src/core/tsi/transport_security.h', + 'src/core/tsi/transport_security_adapter.h', 'src/core/tsi/transport_security_interface.h', 'src/core/ext/transport/chttp2/server/chttp2_server.h', 'src/core/ext/filters/client_channel/client_channel.h', @@ -631,6 +632,7 @@ Pod::Spec.new do |s| 'src/core/tsi/fake_transport_security.c', 'src/core/tsi/ssl_transport_security.c', 'src/core/tsi/transport_security.c', + 'src/core/tsi/transport_security_adapter.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', 'src/core/ext/filters/client_channel/channel_connectivity.c', @@ -857,6 +859,7 @@ Pod::Spec.new do |s| 'src/core/tsi/ssl_transport_security.h', 'src/core/tsi/ssl_types.h', 'src/core/tsi/transport_security.h', + 'src/core/tsi/transport_security_adapter.h', 'src/core/tsi/transport_security_interface.h', 'src/core/ext/transport/chttp2/server/chttp2_server.h', 'src/core/ext/filters/client_channel/client_channel.h', diff --git a/grpc.gemspec b/grpc.gemspec index 6a204685a03..abbd6f42518 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -323,6 +323,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/tsi/ssl_transport_security.h ) s.files += %w( src/core/tsi/ssl_types.h ) s.files += %w( src/core/tsi/transport_security.h ) + s.files += %w( src/core/tsi/transport_security_adapter.h ) s.files += %w( src/core/tsi/transport_security_interface.h ) s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.h ) s.files += %w( src/core/ext/filters/client_channel/client_channel.h ) @@ -547,6 +548,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/tsi/fake_transport_security.c ) s.files += %w( src/core/tsi/ssl_transport_security.c ) s.files += %w( src/core/tsi/transport_security.c ) + s.files += %w( src/core/tsi/transport_security_adapter.c ) s.files += %w( src/core/ext/transport/chttp2/server/chttp2_server.c ) s.files += %w( src/core/ext/transport/chttp2/client/secure/secure_channel_create.c ) s.files += %w( src/core/ext/filters/client_channel/channel_connectivity.c ) diff --git a/package.xml b/package.xml index 6957ecca731..7a301aa8300 100644 --- a/package.xml +++ b/package.xml @@ -332,6 +332,7 @@ + @@ -556,6 +557,7 @@ + diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index a9cb6a107ca..aa8808ab74c 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -182,7 +182,7 @@ tsi_result tsi_handshaker_create_frame_protector( result = self->vtable->create_frame_protector(self, max_protected_frame_size, protector); if (result == TSI_OK) { - self->frame_protector_created = 1; + self->frame_protector_created = true; } return result; } @@ -206,7 +206,7 @@ void tsi_handshaker_destroy(tsi_handshaker *self) { /* --- tsi_handshaker_result implementation. --- */ -tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, +tsi_result tsi_handshaker_result_extract_peer(const tsi_handshaker_result *self, tsi_peer *peer) { if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT; memset(peer, 0, sizeof(tsi_peer)); @@ -214,7 +214,7 @@ tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, } tsi_result tsi_handshaker_result_create_frame_protector( - tsi_handshaker_result *self, size_t *max_protected_frame_size, + const tsi_handshaker_result *self, size_t *max_protected_frame_size, tsi_frame_protector **protector) { if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT; return self->vtable->create_frame_protector(self, max_protected_frame_size, @@ -222,7 +222,8 @@ tsi_result tsi_handshaker_result_create_frame_protector( } tsi_result tsi_handshaker_result_get_unused_bytes( - tsi_handshaker_result *self, unsigned char **bytes, size_t *bytes_size) { + const tsi_handshaker_result *self, unsigned char **bytes, + size_t *bytes_size) { if (self == NULL || bytes == NULL || bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } diff --git a/src/core/tsi/transport_security.h b/src/core/tsi/transport_security.h index 2e821108279..a4c9cbc0011 100644 --- a/src/core/tsi/transport_security.h +++ b/src/core/tsi/transport_security.h @@ -34,6 +34,8 @@ #ifndef GRPC_CORE_TSI_TRANSPORT_SECURITY_H #define GRPC_CORE_TSI_TRANSPORT_SECURITY_H +#include + #include "src/core/tsi/transport_security_interface.h" #ifdef __cplusplus @@ -90,18 +92,18 @@ typedef struct { struct tsi_handshaker { const tsi_handshaker_vtable *vtable; - int frame_protector_created; - int handshaker_result_created; + bool frame_protector_created; + bool handshaker_result_created; }; /* Base for tsi_handshaker_result implementations. See transport_security_interface.h for documentation. */ typedef struct { - tsi_result (*extract_peer)(tsi_handshaker_result *self, tsi_peer *peer); - tsi_result (*create_frame_protector)(tsi_handshaker_result *self, + tsi_result (*extract_peer)(const tsi_handshaker_result *self, tsi_peer *peer); + tsi_result (*create_frame_protector)(const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector); - tsi_result (*get_unused_bytes)(tsi_handshaker_result *self, + tsi_result (*get_unused_bytes)(const tsi_handshaker_result *self, unsigned char **bytes, size_t *bytes_size); void (*destroy)(tsi_handshaker_result *self); } tsi_handshaker_result_vtable; diff --git a/src/core/tsi/transport_security_adapter. b/src/core/tsi/transport_security_adapter. new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c index 7e0b8f574a0..60cebdd6f25 100644 --- a/src/core/tsi/transport_security_adapter.c +++ b/src/core/tsi/transport_security_adapter.c @@ -50,14 +50,14 @@ typedef struct { size_t unused_bytes_size; } tsi_adapter_handshaker_result; -static tsi_result tsi_adapter_result_extract_peer(tsi_handshaker_result *self, - tsi_peer *peer) { +static tsi_result tsi_adapter_result_extract_peer( + const tsi_handshaker_result *self, tsi_peer *peer) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; return tsi_handshaker_extract_peer(impl->handshaker, peer); } static tsi_result tsi_adapter_result_create_frame_protector( - tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; return tsi_handshaker_create_frame_protector( @@ -65,7 +65,8 @@ static tsi_result tsi_adapter_result_create_frame_protector( } static tsi_result tsi_adapter_result_get_unused_bytes( - tsi_handshaker_result *self, unsigned char **bytes, size_t *byte_size) { + const tsi_handshaker_result *self, unsigned char **bytes, + size_t *byte_size) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; *bytes = impl->unused_bytes; *byte_size = impl->unused_bytes_size; @@ -74,9 +75,7 @@ static tsi_result tsi_adapter_result_get_unused_bytes( static void tsi_adapter_result_destroy(tsi_handshaker_result *self) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; - if (impl->unused_bytes != NULL) { - gpr_free(impl->unused_bytes); - } + gpr_free(impl->unused_bytes); gpr_free(self); } @@ -114,45 +113,45 @@ typedef struct { size_t adapter_buffer_size; } tsi_adapter_handshaker; -tsi_result tsi_adapter_get_bytes_to_send_to_peer(tsi_handshaker *self, - unsigned char *bytes, - size_t *bytes_size) { +static tsi_result tsi_adapter_get_bytes_to_send_to_peer(tsi_handshaker *self, + unsigned char *bytes, + size_t *bytes_size) { return tsi_handshaker_get_bytes_to_send_to_peer( tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); } -tsi_result tsi_adapter_process_bytes_from_peer(tsi_handshaker *self, - const unsigned char *bytes, - size_t *bytes_size) { +static tsi_result tsi_adapter_process_bytes_from_peer( + tsi_handshaker *self, const unsigned char *bytes, size_t *bytes_size) { return tsi_handshaker_process_bytes_from_peer( tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); } -tsi_result tsi_adapter_get_result(tsi_handshaker *self) { +static tsi_result tsi_adapter_get_result(tsi_handshaker *self) { return tsi_handshaker_get_result(tsi_adapter_handshaker_get_wrapped(self)); } -tsi_result tsi_adapter_extract_peer(tsi_handshaker *self, tsi_peer *peer) { +static tsi_result tsi_adapter_extract_peer(tsi_handshaker *self, + tsi_peer *peer) { return tsi_handshaker_extract_peer(tsi_adapter_handshaker_get_wrapped(self), peer); } -tsi_result tsi_adapter_create_frame_protector(tsi_handshaker *self, - size_t *max_protected_frame_size, - tsi_frame_protector **protector) { +static tsi_result tsi_adapter_create_frame_protector( + tsi_handshaker *self, size_t *max_protected_frame_size, + tsi_frame_protector **protector) { return tsi_handshaker_create_frame_protector( tsi_adapter_handshaker_get_wrapped(self), max_protected_frame_size, protector); } -void tsi_adapter_destroy(tsi_handshaker *self) { +static void tsi_adapter_destroy(tsi_handshaker *self) { tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self; tsi_handshaker_destroy(impl->wrapped); gpr_free(impl->adapter_buffer); gpr_free(self); } -tsi_result tsi_adapter_next( +static tsi_result tsi_adapter_next( tsi_handshaker *self, const unsigned char *received_bytes, size_t received_bytes_size, unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, @@ -196,11 +195,15 @@ tsi_result tsi_adapter_next( size_t unused_bytes_size = received_bytes_size - bytes_consumed; const unsigned char *unused_bytes = unused_bytes_size == 0 ? NULL : received_bytes + bytes_consumed; - return tsi_adapter_create_handshaker_result( + status = tsi_adapter_create_handshaker_result( impl->wrapped, unused_bytes, unused_bytes_size, handshaker_result); + if (status == TSI_OK) { + impl->base.handshaker_result_created = true; + } + } else { + *handshaker_result = NULL; } - *handshaker_result = NULL; - return TSI_OK; + return status; } static const tsi_handshaker_vtable handshaker_vtable = { diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index be04810e24f..c9495f80b6d 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -220,22 +220,22 @@ typedef struct tsi_handshaker_result tsi_handshaker_result; /* This method extracts tsi peer. It returns TSI_OK assuming there is no fatal error. The caller is responsible for destructing the peer. */ -tsi_result tsi_handshaker_result_extract_peer(tsi_handshaker_result *self, +tsi_result tsi_handshaker_result_extract_peer(const tsi_handshaker_result *self, tsi_peer *peer); /* This method creates a tsi_frame_protector object. It returns TSI_OK assuming there is no fatal error. The caller is responsible for destroying the protector. */ tsi_result tsi_handshaker_result_create_frame_protector( - tsi_handshaker_result *self, size_t *max_output_protected_frame_size, + const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector); /* This method returns the unused bytes from the handshake. It returns TSI_OK assuming there is no fatal error. The caller should not free the bytes. */ -tsi_result tsi_handshaker_result_get_unused_bytes(tsi_handshaker_result *self, - unsigned char **bytes, - size_t *byte_size); +tsi_result tsi_handshaker_result_get_unused_bytes( + const tsi_handshaker_result *self, unsigned char **bytes, + size_t *byte_size); /* This method releases the tsi_handshaker_handshaker object. After this method is called, no other method can be called on the object. */ @@ -305,8 +305,8 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); ... ------------------------------------------------------------------------ - A typical usage of the new TSI would be as follows, supporting both - synchronous and asynchrnous TSI handshaker implementations: + A typical usage supporting both synchronous and asynchronous TSI handshaker + implementations would be: ------------------------------------------------------------------------ @@ -324,17 +324,6 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); ... } - // This method is a wrapper of the callback function to execute when - // tsi_handshaker_next finishes. It is passed to tsi_handshaker_next as - // the callback function. - void on_handshake_next_done_wrapper( - tsi_result status, void *user_data, const unsigned char *bytes_to_send, - size_t bytes_to_send_size, tsi_handshaker_result *result) { - security_handshaker *h = (security_handshaker *)user_data; - on_handshake_next_done(h, status, bytes_to_send, - bytes_to_send_size, result); - } - // This method is the callback function when there are data received from // the peer. This method will read bytes into the handshake buffer and call // do_handshake_next. @@ -355,19 +344,22 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); tsi_handshaker_result *result = NULL; status = tsi_handshaker_next( handshaker, bytes_received, bytes_received_size, &bytes_to_send, - &bytes_to_send_size, &result, on_handshake_next_done_wrapper, h); + &bytes_to_send_size, &result, on_handshake_next_done, h); // If TSI handshaker is asynchronous, on_handshake_next_done will be - // called during the execution of the callback function. + // executed inside tsi_handshaker_next. if (status == TSI_ASYNC) return; - on_handshake_next_done(h, status, bytes_to_send, + // If TSI handshaker is synchronous, invoke callback directly in this + // thread. + on_handshake_next_done(status, (void *)h, bytes_to_send, bytes_to_send_size, result); } - // This is the real function to execute after tsi_handshaker_next. + // This is the callback function to execute after tsi_handshaker_next. + // It is passed to tsi_handshaker_next as a function parameter. void on_handshake_next_done( - security_handshaker *h, tsi_result status, - const unsigned char *bytes_to_send, size_t bytes_to_send_size, - tsi_handshaker_result *result) { + tsi_result status, void *user_data, const unsigned char *bytes_to_send, + size_t bytes_to_send_size, tsi_handshaker_result *result) { + security_handshaker *h = (security_handshaker *)user_data; if (status == TSI_INCOMPLETE_DATA) { // Schedule an asynchronous read from the peer. If handshake data are // received, on_handshake_data_received_from_peer will be called. @@ -386,7 +378,7 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); // Check the Peer. tsi_peer peer; status = tsi_handshaker_result_extract_peer(result, &peer); - if (status != TSI_OK) return status; + if (status != TSI_OK) return; status = check_peer(&peer); tsi_peer_destruct(&peer); if (status != TSI_OK) return; diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 3bcbe667e2c..88e783815c7 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -250,6 +250,7 @@ CORE_SOURCE_FILES = [ 'src/core/tsi/fake_transport_security.c', 'src/core/tsi/ssl_transport_security.c', 'src/core/tsi/transport_security.c', + 'src/core/tsi/transport_security_adapter.c', 'src/core/ext/transport/chttp2/server/chttp2_server.c', 'src/core/ext/transport/chttp2/client/secure/secure_channel_create.c', 'src/core/ext/filters/client_channel/channel_connectivity.c', diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 6903a0e5d5a..55a707d9197 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1369,6 +1369,8 @@ src/core/tsi/ssl_transport_security.h \ src/core/tsi/ssl_types.h \ src/core/tsi/transport_security.c \ src/core/tsi/transport_security.h \ +src/core/tsi/transport_security_adapter.c \ +src/core/tsi/transport_security_adapter.h \ src/core/tsi/transport_security_interface.h \ third_party/nanopb/pb.h \ third_party/nanopb/pb_common.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 11176d97567..c871256d6ed 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -8578,6 +8578,7 @@ "src/core/tsi/ssl_transport_security.h", "src/core/tsi/ssl_types.h", "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_adapter.h", "src/core/tsi/transport_security_interface.h" ], "is_filegroup": true, @@ -8591,6 +8592,8 @@ "src/core/tsi/ssl_types.h", "src/core/tsi/transport_security.c", "src/core/tsi/transport_security.h", + "src/core/tsi/transport_security_adapter.c", + "src/core/tsi/transport_security_adapter.h", "src/core/tsi/transport_security_interface.h" ], "third_party": false, diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index 948af2f29ee..1fec7fb353d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -452,6 +452,7 @@ + @@ -852,6 +853,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index d43b0e3f83e..28776121ecd 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -523,6 +523,9 @@ src\core\tsi + + src\core\tsi + src\core\ext\transport\chttp2\server @@ -1253,6 +1256,9 @@ src\core\tsi + + src\core\tsi + src\core\tsi From a2a2024944a92ec1dbf0ec827659ee408cb4c1e3 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Mon, 10 Apr 2017 15:03:21 -0700 Subject: [PATCH 17/60] Use handshaker_result_created in tsi_adapter_handshaker --- src/core/tsi/transport_security_adapter.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c index 60cebdd6f25..ebf02791396 100644 --- a/src/core/tsi/transport_security_adapter.c +++ b/src/core/tsi/transport_security_adapter.c @@ -191,7 +191,9 @@ static tsi_result tsi_adapter_next( *bytes_to_send_size = offset; /* If handshake completes, create tsi_handshaker_result. */ - if (!tsi_handshaker_is_in_progress(impl->wrapped)) { + if (tsi_handshaker_is_in_progress(impl->wrapped)) { + *handshaker_result = NULL; + } else { size_t unused_bytes_size = received_bytes_size - bytes_consumed; const unsigned char *unused_bytes = unused_bytes_size == 0 ? NULL : received_bytes + bytes_consumed; @@ -200,8 +202,6 @@ static tsi_result tsi_adapter_next( if (status == TSI_OK) { impl->base.handshaker_result_created = true; } - } else { - *handshaker_result = NULL; } return status; } From 9f3822c4a873f96c7d1766939cba79e9c73abf6d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 10 Apr 2017 15:07:22 -0700 Subject: [PATCH 18/60] patch ruby stuff --- src/ruby/ext/grpc/extconf.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index 4f5d77012b1..6c0486f1a8c 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -65,6 +65,7 @@ ENV['MACOSX_DEPLOYMENT_TARGET'] = '10.7' ENV['AR'] = RbConfig::CONFIG['AR'] + ' rcs' ENV['CC'] = RbConfig::CONFIG['CC'] +ENV['CXX'] = RbConfig::CONFIG['CXX'] ENV['LD'] = ENV['CC'] ENV['AR'] = 'libtool -o' if RUBY_PLATFORM =~ /darwin/ From 3b90055f2cd5a20d1f4ff2b4d1581795a4eef645 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Mon, 10 Apr 2017 15:53:17 -0700 Subject: [PATCH 19/60] Fix mac build --- src/python/grpcio/commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index e50ccbe23e7..f196b028a76 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -260,7 +260,7 @@ class BuildExt(build_ext.build_ext): """Custom build_ext command to enable compiler-specific flags.""" C_OPTIONS = { - 'unix': ('-pthread', '-std=gnu99'), + 'unix': ('-pthread',), 'msvc': (), } LINK_OPTIONS = {} From 2d111bd2f994c771f8a260cffee9e1c49dc99931 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 09:18:14 -0700 Subject: [PATCH 20/60] sanity --- CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e3e7d080334..008584a5a4e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8982,7 +8982,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_cq_multiple_threads test/cpp/microbenchmarks/bm_cq_multiple_threads.cc - third_party/googletest/src/gtest-all.cc + third_party/googletest/googletest/src/gtest-all.cc ) @@ -8999,8 +8999,8 @@ target_include_directories(bm_cq_multiple_threads PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10739,7 +10739,7 @@ if (gRPC_BUILD_TESTS) add_executable(memory_test test/core/support/memory_test.cc - third_party/googletest/src/gtest-all.cc + third_party/googletest/googletest/src/gtest-all.cc ) @@ -10756,8 +10756,8 @@ target_include_directories(memory_test PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include - PRIVATE third_party/googletest/include - PRIVATE third_party/googletest + PRIVATE third_party/googletest/googletest/include + PRIVATE third_party/googletest/googletest PRIVATE ${_gRPC_PROTO_GENS_DIR} ) From c19550272e4883c30ab50e301617b3e04e8db39a Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Tue, 11 Apr 2017 09:36:19 -0700 Subject: [PATCH 21/60] Revise based on Julien's comments. --- src/core/tsi/transport_security.c | 36 +++++++++ src/core/tsi/transport_security_adapter.c | 62 +++++++------- src/core/tsi/transport_security_adapter.h | 11 ++- src/core/tsi/transport_security_interface.h | 89 ++++----------------- 4 files changed, 93 insertions(+), 105 deletions(-) diff --git a/src/core/tsi/transport_security.c b/src/core/tsi/transport_security.c index aa8808ab74c..b11c00c43c4 100644 --- a/src/core/tsi/transport_security.c +++ b/src/core/tsi/transport_security.c @@ -94,6 +94,9 @@ tsi_result tsi_frame_protector_protect(tsi_frame_protector *self, protected_output_frames_size == NULL) { return TSI_INVALID_ARGUMENT; } + if (self->vtable == NULL || self->vtable->protect == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->protect(self, unprotected_bytes, unprotected_bytes_size, protected_output_frames, protected_output_frames_size); @@ -106,6 +109,9 @@ tsi_result tsi_frame_protector_protect_flush( protected_output_frames_size == NULL || still_pending_size == NULL) { return TSI_INVALID_ARGUMENT; } + if (self->vtable == NULL || self->vtable->protect_flush == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->protect_flush(self, protected_output_frames, protected_output_frames_size, still_pending_size); @@ -120,6 +126,9 @@ tsi_result tsi_frame_protector_unprotect( unprotected_bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } + if (self->vtable == NULL || self->vtable->unprotect == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->unprotect(self, protected_frames_bytes, protected_frames_bytes_size, unprotected_bytes, unprotected_bytes_size); @@ -141,6 +150,9 @@ tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, return TSI_INVALID_ARGUMENT; } if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; + if (self->vtable == NULL || self->vtable->get_bytes_to_send_to_peer == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->get_bytes_to_send_to_peer(self, bytes, bytes_size); } @@ -151,12 +163,18 @@ tsi_result tsi_handshaker_process_bytes_from_peer(tsi_handshaker *self, return TSI_INVALID_ARGUMENT; } if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; + if (self->vtable == NULL || self->vtable->process_bytes_from_peer == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->process_bytes_from_peer(self, bytes, bytes_size); } tsi_result tsi_handshaker_get_result(tsi_handshaker *self) { if (self == NULL) return TSI_INVALID_ARGUMENT; if (self->frame_protector_created) return TSI_FAILED_PRECONDITION; + if (self->vtable == NULL || self->vtable->get_result == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->get_result(self); } @@ -167,6 +185,9 @@ tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer) { if (tsi_handshaker_get_result(self) != TSI_OK) { return TSI_FAILED_PRECONDITION; } + if (self->vtable == NULL || self->vtable->extract_peer == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->extract_peer(self, peer); } @@ -179,6 +200,9 @@ tsi_result tsi_handshaker_create_frame_protector( if (tsi_handshaker_get_result(self) != TSI_OK) { return TSI_FAILED_PRECONDITION; } + if (self->vtable == NULL || self->vtable->create_frame_protector == NULL) { + return TSI_UNIMPLEMENTED; + } result = self->vtable->create_frame_protector(self, max_protected_frame_size, protector); if (result == TSI_OK) { @@ -194,6 +218,9 @@ tsi_result tsi_handshaker_next( tsi_handshaker_on_next_done_cb cb, void *user_data) { if (self == NULL) return TSI_INVALID_ARGUMENT; if (self->handshaker_result_created) return TSI_FAILED_PRECONDITION; + if (self->vtable == NULL || self->vtable->next == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->next(self, received_bytes, received_bytes_size, bytes_to_send, bytes_to_send_size, handshaker_result, cb, user_data); @@ -210,6 +237,9 @@ tsi_result tsi_handshaker_result_extract_peer(const tsi_handshaker_result *self, tsi_peer *peer) { if (self == NULL || peer == NULL) return TSI_INVALID_ARGUMENT; memset(peer, 0, sizeof(tsi_peer)); + if (self->vtable == NULL || self->vtable->extract_peer == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->extract_peer(self, peer); } @@ -217,6 +247,9 @@ tsi_result tsi_handshaker_result_create_frame_protector( const tsi_handshaker_result *self, size_t *max_protected_frame_size, tsi_frame_protector **protector) { if (self == NULL || protector == NULL) return TSI_INVALID_ARGUMENT; + if (self->vtable == NULL || self->vtable->create_frame_protector == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->create_frame_protector(self, max_protected_frame_size, protector); } @@ -227,6 +260,9 @@ tsi_result tsi_handshaker_result_get_unused_bytes( if (self == NULL || bytes == NULL || bytes_size == NULL) { return TSI_INVALID_ARGUMENT; } + if (self->vtable == NULL || self->vtable->get_unused_bytes == NULL) { + return TSI_UNIMPLEMENTED; + } return self->vtable->get_unused_bytes(self, bytes, bytes_size); } diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c index ebf02791396..d83339b3eb5 100644 --- a/src/core/tsi/transport_security_adapter.c +++ b/src/core/tsi/transport_security_adapter.c @@ -45,26 +45,26 @@ typedef struct { tsi_handshaker_result base; - tsi_handshaker *handshaker; + tsi_handshaker *wrapped; unsigned char *unused_bytes; size_t unused_bytes_size; } tsi_adapter_handshaker_result; -static tsi_result tsi_adapter_result_extract_peer( - const tsi_handshaker_result *self, tsi_peer *peer) { +static tsi_result adapter_result_extract_peer(const tsi_handshaker_result *self, + tsi_peer *peer) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; - return tsi_handshaker_extract_peer(impl->handshaker, peer); + return tsi_handshaker_extract_peer(impl->wrapped, peer); } -static tsi_result tsi_adapter_result_create_frame_protector( +static tsi_result adapter_result_create_frame_protector( const tsi_handshaker_result *self, size_t *max_output_protected_frame_size, tsi_frame_protector **protector) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; return tsi_handshaker_create_frame_protector( - impl->handshaker, max_output_protected_frame_size, protector); + impl->wrapped, max_output_protected_frame_size, protector); } -static tsi_result tsi_adapter_result_get_unused_bytes( +static tsi_result adapter_result_get_unused_bytes( const tsi_handshaker_result *self, unsigned char **bytes, size_t *byte_size) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; @@ -73,26 +73,26 @@ static tsi_result tsi_adapter_result_get_unused_bytes( return TSI_OK; } -static void tsi_adapter_result_destroy(tsi_handshaker_result *self) { +static void adapter_result_destroy(tsi_handshaker_result *self) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; gpr_free(impl->unused_bytes); gpr_free(self); } static const tsi_handshaker_result_vtable result_vtable = { - tsi_adapter_result_extract_peer, tsi_adapter_result_create_frame_protector, - tsi_adapter_result_get_unused_bytes, tsi_adapter_result_destroy, + adapter_result_extract_peer, adapter_result_create_frame_protector, + adapter_result_get_unused_bytes, adapter_result_destroy, }; tsi_result tsi_adapter_create_handshaker_result( - tsi_handshaker *handshaker, const unsigned char *unused_bytes, + tsi_handshaker *wrapped, const unsigned char *unused_bytes, size_t unused_bytes_size, tsi_handshaker_result **handshaker_result) { - if (handshaker == NULL || (unused_bytes_size > 0 && unused_bytes == NULL)) { + if (wrapped == NULL || (unused_bytes_size > 0 && unused_bytes == NULL)) { return TSI_INVALID_ARGUMENT; } tsi_adapter_handshaker_result *impl = gpr_zalloc(sizeof(*impl)); impl->base.vtable = &result_vtable; - impl->handshaker = handshaker; + impl->wrapped = wrapped; impl->unused_bytes_size = unused_bytes_size; if (unused_bytes_size > 0) { impl->unused_bytes = gpr_malloc(unused_bytes_size); @@ -113,30 +113,30 @@ typedef struct { size_t adapter_buffer_size; } tsi_adapter_handshaker; -static tsi_result tsi_adapter_get_bytes_to_send_to_peer(tsi_handshaker *self, - unsigned char *bytes, - size_t *bytes_size) { +static tsi_result adapter_get_bytes_to_send_to_peer(tsi_handshaker *self, + unsigned char *bytes, + size_t *bytes_size) { return tsi_handshaker_get_bytes_to_send_to_peer( tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); } -static tsi_result tsi_adapter_process_bytes_from_peer( - tsi_handshaker *self, const unsigned char *bytes, size_t *bytes_size) { +static tsi_result adapter_process_bytes_from_peer(tsi_handshaker *self, + const unsigned char *bytes, + size_t *bytes_size) { return tsi_handshaker_process_bytes_from_peer( tsi_adapter_handshaker_get_wrapped(self), bytes, bytes_size); } -static tsi_result tsi_adapter_get_result(tsi_handshaker *self) { +static tsi_result adapter_get_result(tsi_handshaker *self) { return tsi_handshaker_get_result(tsi_adapter_handshaker_get_wrapped(self)); } -static tsi_result tsi_adapter_extract_peer(tsi_handshaker *self, - tsi_peer *peer) { +static tsi_result adapter_extract_peer(tsi_handshaker *self, tsi_peer *peer) { return tsi_handshaker_extract_peer(tsi_adapter_handshaker_get_wrapped(self), peer); } -static tsi_result tsi_adapter_create_frame_protector( +static tsi_result adapter_create_frame_protector( tsi_handshaker *self, size_t *max_protected_frame_size, tsi_frame_protector **protector) { return tsi_handshaker_create_frame_protector( @@ -144,14 +144,14 @@ static tsi_result tsi_adapter_create_frame_protector( protector); } -static void tsi_adapter_destroy(tsi_handshaker *self) { +static void adapter_destroy(tsi_handshaker *self) { tsi_adapter_handshaker *impl = (tsi_adapter_handshaker *)self; tsi_handshaker_destroy(impl->wrapped); gpr_free(impl->adapter_buffer); gpr_free(self); } -static tsi_result tsi_adapter_next( +static tsi_result adapter_next( tsi_handshaker *self, const unsigned char *received_bytes, size_t received_bytes_size, unsigned char **bytes_to_send, size_t *bytes_to_send_size, tsi_handshaker_result **handshaker_result, @@ -207,13 +207,13 @@ static tsi_result tsi_adapter_next( } static const tsi_handshaker_vtable handshaker_vtable = { - tsi_adapter_get_bytes_to_send_to_peer, - tsi_adapter_process_bytes_from_peer, - tsi_adapter_get_result, - tsi_adapter_extract_peer, - tsi_adapter_create_frame_protector, - tsi_adapter_destroy, - tsi_adapter_next, + adapter_get_bytes_to_send_to_peer, + adapter_process_bytes_from_peer, + adapter_get_result, + adapter_extract_peer, + adapter_create_frame_protector, + adapter_destroy, + adapter_next, }; tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped) { diff --git a/src/core/tsi/transport_security_adapter.h b/src/core/tsi/transport_security_adapter.h index de5aaefd5f1..686618b53b1 100644 --- a/src/core/tsi/transport_security_adapter.h +++ b/src/core/tsi/transport_security_adapter.h @@ -41,11 +41,18 @@ extern "C" { #endif /* Create a tsi handshaker that takes an implementation of old interface and - converts into an implementation of new interface. + converts into an implementation of new interface. In the old interface, + there are get_bytes_to_send_to_peer, process_bytes_from_peer, get_result, + extract_peer, and create_frame_protector. In the new interface, only next + method is needed. See transport_security_interface.h for details. Note that + this tsi adapter handshaker is temporary. It will be removed once TSI has + been fully migrated to the new interface. Ownership of input tsi_handshaker is transferred to this new adapter. */ tsi_handshaker *tsi_create_adapter_handshaker(tsi_handshaker *wrapped); -/* Given a tsi adapter handshaker, return the original wrapped handshaker. */ +/* Given a tsi adapter handshaker, return the original wrapped handshaker. The + adapter still owns the wrapped handshaker which should not be destroyed by + the caller. */ tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter); #ifdef __cplusplus diff --git a/src/core/tsi/transport_security_interface.h b/src/core/tsi/transport_security_interface.h index c9495f80b6d..f2112b62b67 100644 --- a/src/core/tsi/transport_security_interface.h +++ b/src/core/tsi/transport_security_interface.h @@ -232,7 +232,8 @@ tsi_result tsi_handshaker_result_create_frame_protector( /* This method returns the unused bytes from the handshake. It returns TSI_OK assuming there is no fatal error. - The caller should not free the bytes. */ + Ownership of the bytes is retained by the handshaker result. As a + consequence, the caller must not free the bytes. */ tsi_result tsi_handshaker_result_get_unused_bytes( const tsi_handshaker_result *self, unsigned char **bytes, size_t *byte_size); @@ -247,64 +248,6 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); ------------------------------------------------------------------------ - A typical usage of the synchronous TSI handshaker would be: - - ------------------------------------------------------------------------ - tsi_result status = TSI_OK; - unsigned char buf[4096]; - const size_t buf_size = 4906; - size_t bytes_received_size = 0; - unsigned char *bytes_to_send = NULL; - size_t bytes_to_send_size = 0; - tsi_handshaker_result *result = NULL; - - while (1) { - status = tsi_handshaker_next( - handshaker, buf, bytes_received_size, - &bytes_to_send, &bytes_to_send_size, &result, NULL, NULL); - if (status == TSI_INCOMPLETE_DATA) { - // Need more data from the peer. - bytes_received_size = buf_size; - read_bytes_from_peer(buf, &bytes_received_size); - continue; - } - if (status != TSI_OK) return status; - if (bytes_to_send_size > 0) { - send_bytes_to_peer(bytes_to_send, bytes_to_send_size); - } - if (result != NULL) break; - bytes_received_size = buf_size; - read_bytes_from_peer(buf, &bytes_received_size); - } - - // Check the Peer. - tsi_peer peer; - status = tsi_handshaker_result_extract_peer(result, &peer); - if (status != TSI_OK) return status; - status = check_peer(&peer); - tsi_peer_destruct(&peer); - if (status != TSI_OK) return status; - - // Create the protector. - tsi_frame_protector* protector = NULL; - status = tsi_handshaker_result_create_frame_protector(result, NULL, - &protector); - if (status != TSI_OK) return status; - - // Do not forget to unprotect outstanding data if any. - unsigned char *unused_bytes = NULL; - size_t unused_bytes_size = 0; - status = tsi_handshaker_result_get_unused_bytes(result, &unused_bytes, - &unused_bytes_size); - if (status != TSI_OK) return status; - if (unused_bytes_size > 0) { - status = tsi_frame_protector_unprotect(protector, unused_bytes, - unused_bytes_size, ..., ...); - .... - } - ... - ------------------------------------------------------------------------ - A typical usage supporting both synchronous and asynchronous TSI handshaker implementations would be: @@ -324,8 +267,8 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); ... } - // This method is the callback function when there are data received from - // the peer. This method will read bytes into the handshake buffer and call + // This method is the callback function when data is received from the + // peer. This method will read bytes into the handshake buffer and call // do_handshake_next. void on_handshake_data_received_from_peer(void *user_data) { security_handshaker *h = (security_handshaker *)user_data; @@ -396,7 +339,7 @@ void tsi_handshaker_result_destroy(tsi_handshaker_result *self); ------------------------------------------------------------------------ */ typedef struct tsi_handshaker tsi_handshaker; -/* TO BE DEPRECATED SOON. +/* TO BE DEPRECATED SOON. Use tsi_handshaker_next instead. Gets bytes that need to be sent to the peer. - bytes is the buffer that will be written with the data to be sent to the peer. @@ -412,7 +355,7 @@ tsi_result tsi_handshaker_get_bytes_to_send_to_peer(tsi_handshaker *self, unsigned char *bytes, size_t *bytes_size); -/* TO BE DEPRECATED SOON. +/* TO BE DEPRECATED SOON. Use tsi_handshaker_next instead. Processes bytes received from the peer. - bytes is the buffer containing the data. - bytes_size is an input/output parameter specifying the size of the data as @@ -439,14 +382,15 @@ tsi_result tsi_handshaker_get_result(tsi_handshaker *self); #define tsi_handshaker_is_in_progress(h) \ (tsi_handshaker_get_result((h)) == TSI_HANDSHAKE_IN_PROGRESS) -/* TO BE DEPRECATED SOON. +/* TO BE DEPRECATED SOON. Use tsi_handshaker_result_extract_peer instead. This method may return TSI_FAILED_PRECONDITION if tsi_handshaker_is_in_progress returns 1, it returns TSI_OK otherwise assuming the handshaker is not in a fatal error state. The caller is responsible for destructing the peer. */ tsi_result tsi_handshaker_extract_peer(tsi_handshaker *self, tsi_peer *peer); -/* TO BE DEPRECATED SOON. +/* TO BE DEPRECATED SOON. Use tsi_handshaker_result_create_frame_protector + instead. This method creates a tsi_frame_protector object after the handshake phase is done. After this method has been called successfully, the only method that can be called on this object is Destroy. @@ -488,13 +432,14 @@ typedef void (*tsi_handshaker_on_next_done_cb)( TSI handshaker implementation. - user_data is the argument to callback function passed from the caller. This method returns TSI_ASYNC if the TSI handshaker implementation is - asynchronous. It returns TSI_OK if the handshake completes or if there are - data to send to the peer, otherwise returns TSI_INCOMPLETE_DATA which - indicates that this method needs to be called again with more data from the - peer. In case of a fatal error in the handshake, another specific error code - is returned. - The caller is responsible for destroying the handshaker_result. However, the - caller should not free bytes_to_send, as the buffer is owned by the + asynchronous, and in this case, the callback is guaranteed to run in another + thread owned by TSI. It returns TSI_OK if the handshake completes or if + there are data to send to the peer, otherwise returns TSI_INCOMPLETE_DATA + which indicates that this method needs to be called again with more data + from the peer. In case of a fatal error in the handshake, another specific + error code is returned. + The caller is responsible for destroying the handshaker_result. However, + the caller should not free bytes_to_send, as the buffer is owned by the tsi_handshaker object. */ tsi_result tsi_handshaker_next( tsi_handshaker *self, const unsigned char *received_bytes, From 85db77951c29c3441cf37bf32980b6185a2d14d2 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 10:46:31 -0700 Subject: [PATCH 22/60] Work around atomics --- BUILD | 6 ++- build.yaml | 3 ++ include/grpc/impl/codegen/port_platform.h | 6 +++ src/core/lib/support/atomic.h | 45 ++++++++++++++++ src/core/lib/support/atomic_with_atm.h | 66 +++++++++++++++++++++++ src/core/lib/support/atomic_with_std.h | 48 +++++++++++++++++ src/core/lib/surface/lame_client.cc | 8 +-- 7 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 src/core/lib/support/atomic.h create mode 100644 src/core/lib/support/atomic_with_atm.h create mode 100644 src/core/lib/support/atomic_with_std.h diff --git a/BUILD b/BUILD index 1991c867432..c78a991e271 100644 --- a/BUILD +++ b/BUILD @@ -366,6 +366,10 @@ grpc_cc_library( "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", + "src/core/lib/support/memory.h" + "src/core/lib/support/atomic.h" + "src/core/lib/support/atomic_with_atm.h" + "src/core/lib/support/atomic_with_std.h" "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.h", "src/core/lib/support/spinlock.h", @@ -537,7 +541,7 @@ grpc_cc_library( "src/core/lib/surface/completion_queue.c", "src/core/lib/surface/completion_queue_factory.c", "src/core/lib/surface/event_string.c", - "src/core/lib/surface/lame_client.c", + "src/core/lib/surface/lame_client.cc", "src/core/lib/surface/metadata_array.c", "src/core/lib/surface/server.c", "src/core/lib/surface/validate_metadata.c", diff --git a/build.yaml b/build.yaml index e26fa16968a..483f950cb79 100644 --- a/build.yaml +++ b/build.yaml @@ -90,6 +90,9 @@ filegroups: - src/core/lib/support/block_annotate.h - src/core/lib/support/env.h - src/core/lib/support/memory.h + - src/core/lib/support/atomic.h + - src/core/lib/support/atomic_with_atm.h + - src/core/lib/support/atomic_with_std.h - src/core/lib/support/mpscq.h - src/core/lib/support/murmur_hash.h - src/core/lib/support/spinlock.h diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index 813e08b86e3..bac409e514d 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -290,6 +290,12 @@ #endif #endif /* GPR_NO_AUTODETECT_PLATFORM */ +#if defined(__has_include) +#if __has_include() +#define GRPC_HAS_CXX11_ATOMIC +#endif // __has_include() +#endif // defined(__has_include) + #ifndef GPR_PLATFORM_STRING #warning "GPR_PLATFORM_STRING not auto-detected" #define GPR_PLATFORM_STRING "unknown" diff --git a/src/core/lib/support/atomic.h b/src/core/lib/support/atomic.h new file mode 100644 index 00000000000..7cb495a0202 --- /dev/null +++ b/src/core/lib/support/atomic.h @@ -0,0 +1,45 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_H +#define GRPC_CORE_LIB_SUPPORT_ATOMIC_H + +#include + +#ifdef GPR_HAS_CXX11_ATOMIC +#include "atomic_with_std.h" +#else +#include "atomic_with_atm.h" +#endif + +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */ diff --git a/src/core/lib/support/atomic_with_atm.h b/src/core/lib/support/atomic_with_atm.h new file mode 100644 index 00000000000..6ad375736f9 --- /dev/null +++ b/src/core/lib/support/atomic_with_atm.h @@ -0,0 +1,66 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H +#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H + +#include + +namespace grpc_core { + +enum MemoryOrderRelaxed { memory_order_relaxed }; + +template +class atomic { + public: + static_assert(sizeof(T) <= sizeof(gpr_atm), + "Atomics of size > sizeof(gpr_atm) are not supported"); + atomic() { gpr_atm_no_barrier_store(&x_, static_cast(T())); } + + bool compare_exchange_strong(T& expected, T update, MemoryOrderRelaxed, + MemoryOrderRelaxed) { + if (!gpr_atm_no_barrier_cas(&x_, static_cast(expected), + static_cast(update))) { + expected = static_cast(gpr_atm_no_barrier_load(&x_)); + return false; + } + return true; + } + + private: + gpr_atm x_; +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */ diff --git a/src/core/lib/support/atomic_with_std.h b/src/core/lib/support/atomic_with_std.h new file mode 100644 index 00000000000..7e9c19efe8a --- /dev/null +++ b/src/core/lib/support/atomic_with_std.h @@ -0,0 +1,48 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H +#define GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H + +#include + +namespace grpc_core { + +template +using atomic = std::atomic; + +typedef std::memory_order memory_order; + +} // namespace grpc_core + +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_STD_H */ diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index 4616fc58300..7a64726edb2 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -39,6 +39,8 @@ #include #include +#include "src/core/lib/support/atomic.h" + extern "C" { #include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/support/string.h" @@ -56,7 +58,7 @@ namespace { struct CallData { grpc_linked_mdelem status; grpc_linked_mdelem details; - std::atomic filled_metadata; + grpc_core::atomic filled_metadata; }; struct ChannelData { @@ -69,8 +71,8 @@ static void fill_metadata(grpc_exec_ctx *exec_ctx, grpc_call_element *elem, CallData *calld = static_cast(elem->call_data); bool expected = false; if (!calld->filled_metadata.compare_exchange_strong( - expected, true, std::memory_order_relaxed, - std::memory_order_relaxed)) { + expected, true, grpc_core::memory_order_relaxed, + grpc_core::memory_order_relaxed)) { return; } ChannelData *chand = static_cast(elem->channel_data); From c953b62d396d6168507520d837550f6b12d7e729 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Tue, 11 Apr 2017 10:16:19 -0700 Subject: [PATCH 23/60] Transfer ownership of tsi_handshaker in adapter_create_result. --- src/core/tsi/transport_security_adapter. | 0 src/core/tsi/transport_security_adapter.c | 5 ++++- 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 src/core/tsi/transport_security_adapter. diff --git a/src/core/tsi/transport_security_adapter. b/src/core/tsi/transport_security_adapter. deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/core/tsi/transport_security_adapter.c b/src/core/tsi/transport_security_adapter.c index d83339b3eb5..9f2147b5301 100644 --- a/src/core/tsi/transport_security_adapter.c +++ b/src/core/tsi/transport_security_adapter.c @@ -75,6 +75,7 @@ static tsi_result adapter_result_get_unused_bytes( static void adapter_result_destroy(tsi_handshaker_result *self) { tsi_adapter_handshaker_result *impl = (tsi_adapter_handshaker_result *)self; + tsi_handshaker_destroy(impl->wrapped); gpr_free(impl->unused_bytes); gpr_free(self); } @@ -84,7 +85,8 @@ static const tsi_handshaker_result_vtable result_vtable = { adapter_result_get_unused_bytes, adapter_result_destroy, }; -tsi_result tsi_adapter_create_handshaker_result( +/* Ownership of wrapped tsi_handshaker is transferred to the result object. */ +static tsi_result tsi_adapter_create_handshaker_result( tsi_handshaker *wrapped, const unsigned char *unused_bytes, size_t unused_bytes_size, tsi_handshaker_result **handshaker_result) { if (wrapped == NULL || (unused_bytes_size > 0 && unused_bytes == NULL)) { @@ -201,6 +203,7 @@ static tsi_result adapter_next( impl->wrapped, unused_bytes, unused_bytes_size, handshaker_result); if (status == TSI_OK) { impl->base.handshaker_result_created = true; + impl->wrapped = NULL; } } return status; From c611e60da200ee7ee1b8c5b54ab41c8386284a7e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 19:03:42 -0700 Subject: [PATCH 24/60] Fix include --- src/core/lib/surface/lame_client.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/lib/surface/lame_client.cc b/src/core/lib/surface/lame_client.cc index 7a64726edb2..88f4eaac08f 100644 --- a/src/core/lib/surface/lame_client.cc +++ b/src/core/lib/surface/lame_client.cc @@ -34,7 +34,6 @@ #include #include -#include #include #include From a566685f99d423f34c69678b2b7cbca85d974210 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Tue, 11 Apr 2017 22:20:22 -0700 Subject: [PATCH 25/60] Public headers must be c89 --- include/grpc/impl/codegen/port_platform.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index bac409e514d..a828f6c188f 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -293,8 +293,8 @@ #if defined(__has_include) #if __has_include() #define GRPC_HAS_CXX11_ATOMIC -#endif // __has_include() -#endif // defined(__has_include) +#endif /* __has_include() */ +#endif /* defined(__has_include) */ #ifndef GPR_PLATFORM_STRING #warning "GPR_PLATFORM_STRING not auto-detected" From 90ce723aca9d1527d410bd36f14b40424b0c6534 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 06:53:09 -0700 Subject: [PATCH 26/60] Fixes --- build.yaml | 6 +++--- gRPC-Core.podspec | 6 ++++++ grpc.gemspec | 3 +++ include/grpc/impl/codegen/port_platform.h | 4 ++-- package.xml | 3 +++ src/core/lib/support/atomic_with_atm.h | 2 +- tools/doxygen/Doxyfile.core.internal | 3 +++ tools/run_tests/generated/sources_and_headers.json | 6 ++++++ tools/run_tests/run_tests.py | 5 +++-- vsprojects/vcxproj/gpr/gpr.vcxproj | 3 +++ vsprojects/vcxproj/gpr/gpr.vcxproj.filters | 9 +++++++++ 11 files changed, 42 insertions(+), 8 deletions(-) diff --git a/build.yaml b/build.yaml index cf22f19a988..6fa285b5466 100644 --- a/build.yaml +++ b/build.yaml @@ -86,13 +86,13 @@ filegroups: headers: - src/core/lib/profiling/timers.h - src/core/lib/support/arena.h + - src/core/lib/support/atomic.h + - src/core/lib/support/atomic_with_atm.h + - src/core/lib/support/atomic_with_std.h - src/core/lib/support/backoff.h - src/core/lib/support/block_annotate.h - src/core/lib/support/env.h - src/core/lib/support/memory.h - - src/core/lib/support/atomic.h - - src/core/lib/support/atomic_with_atm.h - - src/core/lib/support/atomic_with_std.h - src/core/lib/support/mpscq.h - src/core/lib/support/murmur_hash.h - src/core/lib/support/spinlock.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index db6a4974c78..5d9063d9ece 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -197,6 +197,9 @@ Pod::Spec.new do |s| # To save you from scrolling, this is the last part of the podspec. ss.source_files = 'src/core/lib/profiling/timers.h', 'src/core/lib/support/arena.h', + 'src/core/lib/support/atomic.h', + 'src/core/lib/support/atomic_with_atm.h', + 'src/core/lib/support/atomic_with_std.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', @@ -696,6 +699,9 @@ Pod::Spec.new do |s| ss.private_header_files = 'src/core/lib/profiling/timers.h', 'src/core/lib/support/arena.h', + 'src/core/lib/support/atomic.h', + 'src/core/lib/support/atomic_with_atm.h', + 'src/core/lib/support/atomic_with_std.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', diff --git a/grpc.gemspec b/grpc.gemspec index cb6c51f898e..0889f2f0c2b 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -82,6 +82,9 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_windows.h ) s.files += %w( src/core/lib/profiling/timers.h ) s.files += %w( src/core/lib/support/arena.h ) + s.files += %w( src/core/lib/support/atomic.h ) + s.files += %w( src/core/lib/support/atomic_with_atm.h ) + s.files += %w( src/core/lib/support/atomic_with_std.h ) s.files += %w( src/core/lib/support/backoff.h ) s.files += %w( src/core/lib/support/block_annotate.h ) s.files += %w( src/core/lib/support/env.h ) diff --git a/include/grpc/impl/codegen/port_platform.h b/include/grpc/impl/codegen/port_platform.h index a828f6c188f..e12f6f4e998 100644 --- a/include/grpc/impl/codegen/port_platform.h +++ b/include/grpc/impl/codegen/port_platform.h @@ -293,8 +293,8 @@ #if defined(__has_include) #if __has_include() #define GRPC_HAS_CXX11_ATOMIC -#endif /* __has_include() */ -#endif /* defined(__has_include) */ +#endif /* __has_include() */ +#endif /* defined(__has_include) */ #ifndef GPR_PLATFORM_STRING #warning "GPR_PLATFORM_STRING not auto-detected" diff --git a/package.xml b/package.xml index 247e4ee76a8..6e601b4d984 100644 --- a/package.xml +++ b/package.xml @@ -91,6 +91,9 @@ + + + diff --git a/src/core/lib/support/atomic_with_atm.h b/src/core/lib/support/atomic_with_atm.h index 6ad375736f9..e7fea018781 100644 --- a/src/core/lib/support/atomic_with_atm.h +++ b/src/core/lib/support/atomic_with_atm.h @@ -63,4 +63,4 @@ class atomic { } // namespace grpc_core -#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */ +#endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_WITH_ATM_H */ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index eb45ba7c0ad..e5cbfb7560d 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1246,6 +1246,9 @@ src/core/lib/support/alloc.c \ src/core/lib/support/arena.c \ src/core/lib/support/arena.h \ src/core/lib/support/atm.c \ +src/core/lib/support/atomic.h \ +src/core/lib/support/atomic_with_atm.h \ +src/core/lib/support/atomic_with_std.h \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/backoff.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 427976c797c..2de4acb2ffe 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7397,6 +7397,9 @@ "include/grpc/support/useful.h", "src/core/lib/profiling/timers.h", "src/core/lib/support/arena.h", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", @@ -7448,6 +7451,9 @@ "src/core/lib/support/arena.c", "src/core/lib/support/arena.h", "src/core/lib/support/atm.c", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/avl.c", "src/core/lib/support/backoff.c", "src/core/lib/support/backoff.h", diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py index defccdc9318..9d3620a0fe3 100755 --- a/tools/run_tests/run_tests.py +++ b/tools/run_tests/run_tests.py @@ -763,7 +763,7 @@ class CSharpLanguage(object): self._make_options = ['EMBED_OPENSSL=true'] if self.args.compiler != 'coreclr': # On Mac, official distribution of mono is 32bit. - self._make_options += ['CFLAGS=-m32', 'CXXFLAGS=-m32', 'LDFLAGS=-m32'] + self._make_options += ['ARCH_FLAGS=-m32', 'LDFLAGS=-m32'] else: self._make_options = ['EMBED_OPENSSL=true', 'EMBED_ZLIB=true'] @@ -1352,7 +1352,8 @@ def make_jobspec(cfg, targets, makefile='Makefile'): '-f', makefile, '-j', '%d' % args.jobs, 'EXTRA_DEFINES=GRPC_TEST_SLOWDOWN_MACHINE_FACTOR=%f' % args.slowdown, - 'CONFIG=%s' % cfg] + + 'CONFIG=%s' % cfg, + 'Q='] + language_make_options + ([] if not args.travis else ['JENKINS_BUILD=1']) + targets, diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj b/vsprojects/vcxproj/gpr/gpr.vcxproj index 474f4d25c9f..7fb81a7fbca 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj @@ -188,6 +188,9 @@ + + + diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters index 760de3070d7..27d9d2f38f4 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters @@ -260,6 +260,15 @@ src\core\lib\support + + src\core\lib\support + + + src\core\lib\support + + + src\core\lib\support + src\core\lib\support From 284195faab8a8b6c5232f722f298f3a51973d64d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 08:29:55 -0700 Subject: [PATCH 27/60] Write out atomic explicitly --- src/core/lib/support/atomic_with_atm.h | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/core/lib/support/atomic_with_atm.h b/src/core/lib/support/atomic_with_atm.h index e7fea018781..55727f1dee2 100644 --- a/src/core/lib/support/atomic_with_atm.h +++ b/src/core/lib/support/atomic_with_atm.h @@ -41,17 +41,21 @@ namespace grpc_core { enum MemoryOrderRelaxed { memory_order_relaxed }; template -class atomic { +class atomic; + +template <> +class atomic { public: - static_assert(sizeof(T) <= sizeof(gpr_atm), - "Atomics of size > sizeof(gpr_atm) are not supported"); - atomic() { gpr_atm_no_barrier_store(&x_, static_cast(T())); } + atomic() { gpr_atm_no_barrier_store(&x_, static_cast(false)); } + explicit atomic(bool x) { + gpr_atm_no_barrier_store(&x_, static_cast(x)); + } - bool compare_exchange_strong(T& expected, T update, MemoryOrderRelaxed, + bool compare_exchange_strong(bool& expected, bool update, MemoryOrderRelaxed, MemoryOrderRelaxed) { if (!gpr_atm_no_barrier_cas(&x_, static_cast(expected), static_cast(update))) { - expected = static_cast(gpr_atm_no_barrier_load(&x_)); + expected = gpr_atm_no_barrier_load(&x_) != 0; return false; } return true; From e85e95022099a88212c87dab48a304b23c2288f8 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 08:35:49 -0700 Subject: [PATCH 28/60] bazelness --- BUILD | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/BUILD b/BUILD index 9f4614ec932..ca8d6051f63 100644 --- a/BUILD +++ b/BUILD @@ -366,10 +366,10 @@ grpc_cc_library( "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", - "src/core/lib/support/memory.h" - "src/core/lib/support/atomic.h" - "src/core/lib/support/atomic_with_atm.h" - "src/core/lib/support/atomic_with_std.h" + "src/core/lib/support/memory.h", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/mpscq.h", "src/core/lib/support/murmur_hash.h", "src/core/lib/support/spinlock.h", From 423d6fd7ed418aead7d494e8d3c31ecd6e2743f1 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 13:15:45 -0700 Subject: [PATCH 29/60] Optimize framing a little - rely on the fact that data-to-come holds a reference to data-being-written, so there's no need to add a ref for every frame written - provide an 'inlined' version of grpc_slice_malloc (via a #define) that gives the compiler more information about small allocations to enable better optimization --- include/grpc/slice.h | 10 ++ include/grpc/slice_buffer.h | 4 + .../lb_policy/grpclb/load_balancer_api.c | 2 +- .../transport/chttp2/transport/bin_decoder.c | 4 +- .../transport/chttp2/transport/bin_encoder.c | 6 +- .../chttp2/transport/chttp2_transport.c | 8 +- .../transport/chttp2/transport/frame_data.c | 4 +- .../transport/chttp2/transport/frame_goaway.c | 2 +- .../transport/chttp2/transport/frame_ping.c | 2 +- .../chttp2/transport/frame_rst_stream.c | 2 +- .../chttp2/transport/frame_settings.c | 4 +- .../chttp2/transport/frame_window_update.c | 2 +- .../chttp2/transport/hpack_encoder.c | 2 +- .../cronet/transport/cronet_transport.c | 2 +- src/core/lib/channel/http_client_filter.c | 2 +- src/core/lib/compression/message_compress.c | 4 +- src/core/lib/iomgr/tcp_windows.c | 2 +- .../lib/security/transport/secure_endpoint.c | 8 +- src/core/lib/slice/b64.c | 2 +- src/core/lib/slice/percent_encoding.c | 6 +- src/core/lib/slice/slice.c | 95 +++++++++++++------ src/core/lib/slice/slice_buffer.c | 36 ++++++- src/core/lib/surface/byte_buffer_reader.c | 2 +- 23 files changed, 150 insertions(+), 61 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index ea66e094e98..4dce6da9155 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -75,6 +75,13 @@ GPRAPI grpc_slice grpc_slice_new_with_len(void *p, size_t len, call. Aborts if malloc() fails. */ GPRAPI grpc_slice grpc_slice_malloc(size_t length); +GPRAPI grpc_slice grpc_slice_malloc_large(size_t length); + +#define GRPC_SLICE_MALLOC(len) \ + ((len) <= GRPC_SLICE_INLINED_SIZE \ + ? (grpc_slice){.refcount = NULL, \ + .data.inlined = {.length = (uint8_t)(len)}} \ + : grpc_slice_malloc_large((len))) /* Intern a slice: @@ -117,6 +124,9 @@ GPRAPI grpc_slice grpc_slice_sub_no_ref(grpc_slice s, size_t begin, size_t end); Requires s intialized, split <= s.length */ GPRAPI grpc_slice grpc_slice_split_tail(grpc_slice *s, size_t split); +/* The same as grpc_slice_split_tail, but without altering refcounts */ +GPRAPI grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *s, size_t split); + /* Splits s into two: modifies s to be s[split:s.length], and returns a new slice, sharing a refcount with s, that contains s[0:split]. Requires s intialized, split <= s.length */ diff --git a/include/grpc/slice_buffer.h b/include/grpc/slice_buffer.h index 2ed896645b4..cdbd74776c6 100644 --- a/include/grpc/slice_buffer.h +++ b/include/grpc/slice_buffer.h @@ -77,6 +77,10 @@ GPRAPI void grpc_slice_buffer_trim_end(grpc_slice_buffer *src, size_t n, /* move the first n bytes of src into dst */ GPRAPI void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst); +/* move the first n bytes of src into dst without adding references */ +GPRAPI void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src, + size_t n, + grpc_slice_buffer *dst); /* move the first n bytes of src into dst (copying them) */ GPRAPI void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx, grpc_slice_buffer *src, diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c index 10af2525313..87549b78f0e 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.c @@ -98,7 +98,7 @@ grpc_slice grpc_grpclb_request_encode(const grpc_grpclb_request *request) { pb_encode(&sizestream, grpc_lb_v1_LoadBalanceRequest_fields, request); encoded_length = sizestream.bytes_written; - slice = grpc_slice_malloc(encoded_length); + slice = GRPC_SLICE_MALLOC(encoded_length); outputstream = pb_ostream_from_buffer(GRPC_SLICE_START_PTR(slice), encoded_length); GPR_ASSERT(pb_encode(&outputstream, grpc_lb_v1_LoadBalanceRequest_fields, diff --git a/src/core/ext/transport/chttp2/transport/bin_decoder.c b/src/core/ext/transport/chttp2/transport/bin_decoder.c index 8c87de112eb..21bed18c9a7 100644 --- a/src/core/ext/transport/chttp2/transport/bin_decoder.c +++ b/src/core/ext/transport/chttp2/transport/bin_decoder.c @@ -169,7 +169,7 @@ grpc_slice grpc_chttp2_base64_decode(grpc_exec_ctx *exec_ctx, } } } - output = grpc_slice_malloc(output_length); + output = GRPC_SLICE_MALLOC(output_length); ctx.input_cur = GRPC_SLICE_START_PTR(input); ctx.input_end = GRPC_SLICE_END_PTR(input); @@ -193,7 +193,7 @@ grpc_slice grpc_chttp2_base64_decode_with_length(grpc_exec_ctx *exec_ctx, grpc_slice input, size_t output_length) { size_t input_length = GRPC_SLICE_LENGTH(input); - grpc_slice output = grpc_slice_malloc(output_length); + grpc_slice output = GRPC_SLICE_MALLOC(output_length); struct grpc_base64_decode_context ctx; // The length of a base64 string cannot be 4 * n + 1 diff --git a/src/core/ext/transport/chttp2/transport/bin_encoder.c b/src/core/ext/transport/chttp2/transport/bin_encoder.c index e301c073f37..3b8ab1f17aa 100644 --- a/src/core/ext/transport/chttp2/transport/bin_encoder.c +++ b/src/core/ext/transport/chttp2/transport/bin_encoder.c @@ -66,7 +66,7 @@ grpc_slice grpc_chttp2_base64_encode(grpc_slice input) { size_t input_triplets = input_length / 3; size_t tail_case = input_length % 3; size_t output_length = input_triplets * 4 + tail_xtra[tail_case]; - grpc_slice output = grpc_slice_malloc(output_length); + grpc_slice output = GRPC_SLICE_MALLOC(output_length); uint8_t *in = GRPC_SLICE_START_PTR(input); char *out = (char *)GRPC_SLICE_START_PTR(output); size_t i; @@ -119,7 +119,7 @@ grpc_slice grpc_chttp2_huffman_compress(grpc_slice input) { nbits += grpc_chttp2_huffsyms[*in].length; } - output = grpc_slice_malloc(nbits / 8 + (nbits % 8 != 0)); + output = GRPC_SLICE_MALLOC(nbits / 8 + (nbits % 8 != 0)); out = GRPC_SLICE_START_PTR(output); for (in = GRPC_SLICE_START_PTR(input); in != GRPC_SLICE_END_PTR(input); ++in) { @@ -184,7 +184,7 @@ grpc_slice grpc_chttp2_base64_encode_and_huffman_compress(grpc_slice input) { size_t output_syms = input_triplets * 4 + tail_xtra[tail_case]; size_t max_output_bits = 11 * output_syms; size_t max_output_length = max_output_bits / 8 + (max_output_bits % 8 != 0); - grpc_slice output = grpc_slice_malloc(max_output_length); + grpc_slice output = GRPC_SLICE_MALLOC(max_output_length); uint8_t *in = GRPC_SLICE_START_PTR(input); uint8_t *start_out = GRPC_SLICE_START_PTR(output); huff_out out; diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index e2816b0e045..f7d78026b7b 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -1906,7 +1906,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, the time we got around to sending this, so instead we ignore HPACK compression and just write the uncompressed bytes onto the wire. */ if (!s->sent_initial_metadata) { - http_status_hdr = grpc_slice_malloc(13); + http_status_hdr = GRPC_SLICE_MALLOC(13); p = GRPC_SLICE_START_PTR(http_status_hdr); *p++ = 0x00; *p++ = 7; @@ -1925,7 +1925,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, len += (uint32_t)GRPC_SLICE_LENGTH(http_status_hdr); } - status_hdr = grpc_slice_malloc(15 + (grpc_status >= 10)); + status_hdr = GRPC_SLICE_MALLOC(15 + (grpc_status >= 10)); p = GRPC_SLICE_START_PTR(status_hdr); *p++ = 0x00; /* literal header, not indexed */ *p++ = 11; /* len(grpc-status) */ @@ -1954,7 +1954,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, size_t msg_len = GRPC_SLICE_LENGTH(slice); GPR_ASSERT(msg_len <= UINT32_MAX); uint32_t msg_len_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)msg_len, 1); - message_pfx = grpc_slice_malloc(14 + msg_len_len); + message_pfx = GRPC_SLICE_MALLOC(14 + msg_len_len); p = GRPC_SLICE_START_PTR(message_pfx); *p++ = 0x00; /* literal header, not indexed */ *p++ = 12; /* len(grpc-message) */ @@ -1976,7 +1976,7 @@ static void close_from_api(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport *t, len += (uint32_t)GRPC_SLICE_LENGTH(message_pfx); len += (uint32_t)msg_len; - hdr = grpc_slice_malloc(9); + hdr = GRPC_SLICE_MALLOC(9); p = GRPC_SLICE_START_PTR(hdr); *p++ = (uint8_t)(len >> 16); *p++ = (uint8_t)(len >> 8); diff --git a/src/core/ext/transport/chttp2/transport/frame_data.c b/src/core/ext/transport/chttp2/transport/frame_data.c index 6e9258ee7ee..e9cf5324b0c 100644 --- a/src/core/ext/transport/chttp2/transport/frame_data.c +++ b/src/core/ext/transport/chttp2/transport/frame_data.c @@ -123,7 +123,7 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, uint8_t *p; static const size_t header_size = 9; - hdr = grpc_slice_malloc(header_size); + hdr = GRPC_SLICE_MALLOC(header_size); p = GRPC_SLICE_START_PTR(hdr); GPR_ASSERT(write_bytes < (1 << 24)); *p++ = (uint8_t)(write_bytes >> 16); @@ -137,7 +137,7 @@ void grpc_chttp2_encode_data(uint32_t id, grpc_slice_buffer *inbuf, *p++ = (uint8_t)(id); grpc_slice_buffer_add(outbuf, hdr); - grpc_slice_buffer_move_first(inbuf, write_bytes, outbuf); + grpc_slice_buffer_move_first_no_ref(inbuf, write_bytes, outbuf); stats->framing_bytes += header_size; stats->data_bytes += write_bytes; diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.c b/src/core/ext/transport/chttp2/transport/frame_goaway.c index 001271dd228..0f1c8b0772f 100644 --- a/src/core/ext/transport/chttp2/transport/frame_goaway.c +++ b/src/core/ext/transport/chttp2/transport/frame_goaway.c @@ -163,7 +163,7 @@ grpc_error *grpc_chttp2_goaway_parser_parse(grpc_exec_ctx *exec_ctx, void grpc_chttp2_goaway_append(uint32_t last_stream_id, uint32_t error_code, grpc_slice debug_data, grpc_slice_buffer *slice_buffer) { - grpc_slice header = grpc_slice_malloc(9 + 4 + 4); + grpc_slice header = GRPC_SLICE_MALLOC(9 + 4 + 4); uint8_t *p = GRPC_SLICE_START_PTR(header); uint32_t frame_length; GPR_ASSERT(GRPC_SLICE_LENGTH(debug_data) < UINT32_MAX - 4 - 4); diff --git a/src/core/ext/transport/chttp2/transport/frame_ping.c b/src/core/ext/transport/chttp2/transport/frame_ping.c index 6016e43127c..f09ca607397 100644 --- a/src/core/ext/transport/chttp2/transport/frame_ping.c +++ b/src/core/ext/transport/chttp2/transport/frame_ping.c @@ -43,7 +43,7 @@ static bool g_disable_ping_ack = false; grpc_slice grpc_chttp2_ping_create(uint8_t ack, uint64_t opaque_8bytes) { - grpc_slice slice = grpc_slice_malloc(9 + 8); + grpc_slice slice = GRPC_SLICE_MALLOC(9 + 8); uint8_t *p = GRPC_SLICE_START_PTR(slice); *p++ = 0; diff --git a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c index 225f15c77c9..e0caa50e92e 100644 --- a/src/core/ext/transport/chttp2/transport/frame_rst_stream.c +++ b/src/core/ext/transport/chttp2/transport/frame_rst_stream.c @@ -44,7 +44,7 @@ grpc_slice grpc_chttp2_rst_stream_create(uint32_t id, uint32_t code, grpc_transport_one_way_stats *stats) { static const size_t frame_size = 13; - grpc_slice slice = grpc_slice_malloc(frame_size); + grpc_slice slice = GRPC_SLICE_MALLOC(frame_size); stats->framing_bytes += frame_size; uint8_t *p = GRPC_SLICE_START_PTR(slice); diff --git a/src/core/ext/transport/chttp2/transport/frame_settings.c b/src/core/ext/transport/chttp2/transport/frame_settings.c index 16881c0707a..2a2986301d2 100644 --- a/src/core/ext/transport/chttp2/transport/frame_settings.c +++ b/src/core/ext/transport/chttp2/transport/frame_settings.c @@ -93,7 +93,7 @@ grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, n += (new[i] != old[i] || (force_mask & (1u << i)) != 0); } - output = grpc_slice_malloc(9 + 6 * n); + output = GRPC_SLICE_MALLOC(9 + 6 * n); p = fill_header(GRPC_SLICE_START_PTR(output), 6 * n, 0); for (i = 0; i < count; i++) { @@ -115,7 +115,7 @@ grpc_slice grpc_chttp2_settings_create(uint32_t *old, const uint32_t *new, } grpc_slice grpc_chttp2_settings_ack_create(void) { - grpc_slice output = grpc_slice_malloc(9); + grpc_slice output = GRPC_SLICE_MALLOC(9); fill_header(GRPC_SLICE_START_PTR(output), 0, GRPC_CHTTP2_FLAG_ACK); return output; } diff --git a/src/core/ext/transport/chttp2/transport/frame_window_update.c b/src/core/ext/transport/chttp2/transport/frame_window_update.c index b76b6f6f477..8ed72dddca0 100644 --- a/src/core/ext/transport/chttp2/transport/frame_window_update.c +++ b/src/core/ext/transport/chttp2/transport/frame_window_update.c @@ -41,7 +41,7 @@ grpc_slice grpc_chttp2_window_update_create( uint32_t id, uint32_t window_update, grpc_transport_one_way_stats *stats) { static const size_t frame_size = 13; - grpc_slice slice = grpc_slice_malloc(frame_size); + grpc_slice slice = GRPC_SLICE_MALLOC(frame_size); stats->header_bytes += frame_size; uint8_t *p = GRPC_SLICE_START_PTR(slice); diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.c b/src/core/ext/transport/chttp2/transport/hpack_encoder.c index 84586cd9988..b9b0f5cab0e 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.c +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.c @@ -122,7 +122,7 @@ static void finish_frame(framer_state *st, int is_header_boundary, output before beginning */ static void begin_frame(framer_state *st) { st->header_idx = - grpc_slice_buffer_add_indexed(st->output, grpc_slice_malloc(9)); + grpc_slice_buffer_add_indexed(st->output, GRPC_SLICE_MALLOC(9)); st->output_length_at_start_of_frame = st->output->length; } diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 0b9189558f8..c0b09e752ca 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -1165,7 +1165,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, } else if (stream_state->rs.remaining_bytes == 0) { CRONET_LOG(GPR_DEBUG, "read operation complete"); grpc_slice read_data_slice = - grpc_slice_malloc((uint32_t)stream_state->rs.length_field); + GRPC_SLICE_MALLOC((uint32_t)stream_state->rs.length_field); uint8_t *dst_p = GRPC_SLICE_START_PTR(read_data_slice); memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); diff --git a/src/core/lib/channel/http_client_filter.c b/src/core/lib/channel/http_client_filter.c index 4e47c5c658f..570ec8608ba 100644 --- a/src/core/lib/channel/http_client_filter.c +++ b/src/core/lib/channel/http_client_filter.c @@ -312,7 +312,7 @@ static grpc_error *hc_mutate_op(grpc_exec_ctx *exec_ctx, op->payload->send_message.send_message->length, k_url_safe, k_multi_line); estimated_len += 1; /* for the trailing 0 */ - grpc_slice path_with_query_slice = grpc_slice_malloc(estimated_len); + grpc_slice path_with_query_slice = GRPC_SLICE_MALLOC(estimated_len); /* memcopy individual pieces into this slice */ uint8_t *write_ptr = diff --git a/src/core/lib/compression/message_compress.c b/src/core/lib/compression/message_compress.c index 49beb953b00..fd3d1e6fcc0 100644 --- a/src/core/lib/compression/message_compress.c +++ b/src/core/lib/compression/message_compress.c @@ -50,7 +50,7 @@ static int zlib_body(grpc_exec_ctx* exec_ctx, z_stream* zs, int r; int flush; size_t i; - grpc_slice outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE); + grpc_slice outbuf = GRPC_SLICE_MALLOC(OUTPUT_BLOCK_SIZE); const uInt uint_max = ~(uInt)0; GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max); @@ -65,7 +65,7 @@ static int zlib_body(grpc_exec_ctx* exec_ctx, z_stream* zs, do { if (zs->avail_out == 0) { grpc_slice_buffer_add_indexed(output, outbuf); - outbuf = grpc_slice_malloc(OUTPUT_BLOCK_SIZE); + outbuf = GRPC_SLICE_MALLOC(OUTPUT_BLOCK_SIZE); GPR_ASSERT(GRPC_SLICE_LENGTH(outbuf) <= uint_max); zs->avail_out = (uInt)GRPC_SLICE_LENGTH(outbuf); zs->next_out = GRPC_SLICE_START_PTR(outbuf); diff --git a/src/core/lib/iomgr/tcp_windows.c b/src/core/lib/iomgr/tcp_windows.c index f74aa687936..bdd4dd07afc 100644 --- a/src/core/lib/iomgr/tcp_windows.c +++ b/src/core/lib/iomgr/tcp_windows.c @@ -219,7 +219,7 @@ static void win_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, tcp->read_slices = read_slices; grpc_slice_buffer_reset_and_unref_internal(exec_ctx, read_slices); - tcp->read_slice = grpc_slice_malloc(8192); + tcp->read_slice = GRPC_SLICE_MALLOC(8192); buffer.len = (ULONG)GRPC_SLICE_LENGTH( tcp->read_slice); // we know slice size fits in 32bit. diff --git a/src/core/lib/security/transport/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c index 24da949e486..0d5c7432c64 100644 --- a/src/core/lib/security/transport/secure_endpoint.c +++ b/src/core/lib/security/transport/secure_endpoint.c @@ -130,7 +130,7 @@ static void secure_endpoint_ref(secure_endpoint *ep) { gpr_ref(&ep->ref); } static void flush_read_staging_buffer(secure_endpoint *ep, uint8_t **cur, uint8_t **end) { grpc_slice_buffer_add(ep->read_buffer, ep->read_staging_buffer); - ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->read_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); *cur = GRPC_SLICE_START_PTR(ep->read_staging_buffer); *end = GRPC_SLICE_END_PTR(ep->read_staging_buffer); } @@ -252,7 +252,7 @@ static void endpoint_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *secure_ep, static void flush_write_staging_buffer(secure_endpoint *ep, uint8_t **cur, uint8_t **end) { grpc_slice_buffer_add(&ep->output_buffer, ep->write_staging_buffer); - ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->write_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); *cur = GRPC_SLICE_START_PTR(ep->write_staging_buffer); *end = GRPC_SLICE_END_PTR(ep->write_staging_buffer); } @@ -415,8 +415,8 @@ grpc_endpoint *grpc_secure_endpoint_create( grpc_slice_buffer_add(&ep->leftover_bytes, grpc_slice_ref_internal(leftover_slices[i])); } - ep->write_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); - ep->read_staging_buffer = grpc_slice_malloc(STAGING_BUFFER_SIZE); + ep->write_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); + ep->read_staging_buffer = GRPC_SLICE_MALLOC(STAGING_BUFFER_SIZE); grpc_slice_buffer_init(&ep->output_buffer); grpc_slice_buffer_init(&ep->source_buffer); ep->read_buffer = NULL; diff --git a/src/core/lib/slice/b64.c b/src/core/lib/slice/b64.c index 2007cc48109..d9091646e06 100644 --- a/src/core/lib/slice/b64.c +++ b/src/core/lib/slice/b64.c @@ -202,7 +202,7 @@ static int decode_group(const unsigned char *codes, size_t num_codes, grpc_slice grpc_base64_decode_with_len(grpc_exec_ctx *exec_ctx, const char *b64, size_t b64_len, int url_safe) { - grpc_slice result = grpc_slice_malloc(b64_len); + grpc_slice result = GRPC_SLICE_MALLOC(b64_len); unsigned char *current = GRPC_SLICE_START_PTR(result); size_t result_size = 0; unsigned char codes[4]; diff --git a/src/core/lib/slice/percent_encoding.c b/src/core/lib/slice/percent_encoding.c index c76c58d371e..a77c69763f1 100644 --- a/src/core/lib/slice/percent_encoding.c +++ b/src/core/lib/slice/percent_encoding.c @@ -71,7 +71,7 @@ grpc_slice grpc_percent_encode_slice(grpc_slice slice, return grpc_slice_ref_internal(slice); } // second pass: actually encode - grpc_slice out = grpc_slice_malloc(output_length); + grpc_slice out = GRPC_SLICE_MALLOC(output_length); uint8_t *q = GRPC_SLICE_START_PTR(out); for (p = slice_start; p < slice_end; p++) { if (is_unreserved_character(*p, unreserved_bytes)) { @@ -125,7 +125,7 @@ bool grpc_strict_percent_decode_slice(grpc_slice slice_in, return true; } p = GRPC_SLICE_START_PTR(slice_in); - *slice_out = grpc_slice_malloc(out_length); + *slice_out = GRPC_SLICE_MALLOC(out_length); uint8_t *q = GRPC_SLICE_START_PTR(*slice_out); while (p != in_end) { if (*p == '%') { @@ -163,7 +163,7 @@ grpc_slice grpc_permissive_percent_decode_slice(grpc_slice slice_in) { return grpc_slice_ref_internal(slice_in); } p = GRPC_SLICE_START_PTR(slice_in); - grpc_slice out = grpc_slice_malloc(out_length); + grpc_slice out = GRPC_SLICE_MALLOC(out_length); uint8_t *q = GRPC_SLICE_START_PTR(out); while (p != in_end) { if (*p == '%') { diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 1cddf062cd4..8896f1dfd3f 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -197,7 +197,7 @@ grpc_slice grpc_slice_new_with_len(void *p, size_t len, } grpc_slice grpc_slice_from_copied_buffer(const char *source, size_t length) { - grpc_slice slice = grpc_slice_malloc(length); + grpc_slice slice = GRPC_SLICE_MALLOC(length); memcpy(GRPC_SLICE_START_PTR(slice), source, length); return slice; } @@ -227,35 +227,42 @@ static const grpc_slice_refcount_vtable malloc_vtable = { malloc_ref, malloc_unref, grpc_slice_default_eq_impl, grpc_slice_default_hash_impl}; +grpc_slice grpc_slice_malloc_large(size_t length) { + grpc_slice slice; + + /* Memory layout used by the slice created here: + + +-----------+----------------------------------------------------------+ + | refcount | bytes | + +-----------+----------------------------------------------------------+ + + refcount is a malloc_refcount + bytes is an array of bytes of the requested length + Both parts are placed in the same allocation returned from gpr_malloc */ + malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length); + + /* Initial refcount on rc is 1 - and it's up to the caller to release + this reference. */ + gpr_ref_init(&rc->refs, 1); + + rc->base.vtable = &malloc_vtable; + rc->base.sub_refcount = &rc->base; + + /* Build up the slice to be returned. */ + /* The slices refcount points back to the allocated block. */ + slice.refcount = &rc->base; + /* The data bytes are placed immediately after the refcount struct */ + slice.data.refcounted.bytes = (uint8_t *)(rc + 1); + /* And the length of the block is set to the requested length */ + slice.data.refcounted.length = length; + return slice; +} + grpc_slice grpc_slice_malloc(size_t length) { grpc_slice slice; if (length > sizeof(slice.data.inlined.bytes)) { - /* Memory layout used by the slice created here: - - +-----------+----------------------------------------------------------+ - | refcount | bytes | - +-----------+----------------------------------------------------------+ - - refcount is a malloc_refcount - bytes is an array of bytes of the requested length - Both parts are placed in the same allocation returned from gpr_malloc */ - malloc_refcount *rc = gpr_malloc(sizeof(malloc_refcount) + length); - - /* Initial refcount on rc is 1 - and it's up to the caller to release - this reference. */ - gpr_ref_init(&rc->refs, 1); - - rc->base.vtable = &malloc_vtable; - rc->base.sub_refcount = &rc->base; - - /* Build up the slice to be returned. */ - /* The slices refcount points back to the allocated block. */ - slice.refcount = &rc->base; - /* The data bytes are placed immediately after the refcount struct */ - slice.data.refcounted.bytes = (uint8_t *)(rc + 1); - /* And the length of the block is set to the requested length */ - slice.data.refcounted.length = length; + return grpc_slice_malloc_large(length); } else { /* small slice: just inline the data */ slice.refcount = NULL; @@ -341,6 +348,40 @@ grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { return tail; } +grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *source, size_t split) { + grpc_slice tail; + + if (source->refcount == NULL) { + /* inlined data, copy it out */ + GPR_ASSERT(source->data.inlined.length >= split); + tail.refcount = NULL; + tail.data.inlined.length = (uint8_t)(source->data.inlined.length - split); + memcpy(tail.data.inlined.bytes, source->data.inlined.bytes + split, + tail.data.inlined.length); + source->data.inlined.length = (uint8_t)split; + } else { + size_t tail_length = source->data.refcounted.length - split; + GPR_ASSERT(source->data.refcounted.length >= split); + if (tail_length < sizeof(tail.data.inlined.bytes)) { + /* Copy out the bytes - it'll be cheaper than refcounting */ + tail.refcount = NULL; + tail.data.inlined.length = (uint8_t)tail_length; + memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split, + tail_length); + } else { + /* Build the result */ + tail.refcount = &noop_refcount; + /* Point into the source array */ + tail.data.refcounted.bytes = source->data.refcounted.bytes + split; + tail.data.refcounted.length = tail_length; + } + source->refcount = source->refcount->sub_refcount; + source->data.refcounted.length = split; + } + + return tail; +} + grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { grpc_slice head; @@ -457,7 +498,7 @@ int grpc_slice_slice(grpc_slice haystack, grpc_slice needle) { } grpc_slice grpc_slice_dup(grpc_slice a) { - grpc_slice copy = grpc_slice_malloc(GRPC_SLICE_LENGTH(a)); + grpc_slice copy = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(a)); memcpy(GRPC_SLICE_START_PTR(copy), GRPC_SLICE_START_PTR(a), GRPC_SLICE_LENGTH(a)); return copy; diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index c96b9c3b281..a13941e5ac8 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -255,14 +255,47 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src, void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst) { + GPR_ASSERT(src->length >= n); + if (src->length == n) { + grpc_slice_buffer_move_into(src, dst); + return; + } + size_t output_len = dst->length + n; size_t new_input_len = src->length - n; + + while (src->count > 0) { + grpc_slice slice = grpc_slice_buffer_take_first(src); + size_t slice_len = GRPC_SLICE_LENGTH(slice); + if (n > slice_len) { + grpc_slice_buffer_add(dst, slice); + n -= slice_len; + } else if (n == slice_len) { + grpc_slice_buffer_add(dst, slice); + break; + } else { /* n < slice_len */ + grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n)); + GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); + grpc_slice_buffer_add(dst, slice); + break; + } + } + GPR_ASSERT(dst->length == output_len); + GPR_ASSERT(src->length == new_input_len); + GPR_ASSERT(src->count > 0); +} + +void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst) { GPR_ASSERT(src->length >= n); if (src->length == n) { grpc_slice_buffer_move_into(src, dst); return; } + size_t output_len = dst->length + n; + size_t new_input_len = src->length - n; + while (src->count > 0) { grpc_slice slice = grpc_slice_buffer_take_first(src); size_t slice_len = GRPC_SLICE_LENGTH(slice); @@ -273,7 +306,8 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer_add(dst, slice); break; } else { /* n < slice_len */ - grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n)); + grpc_slice_buffer_undo_take_first( + src, grpc_slice_split_tail_no_ref(&slice, n)); GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); grpc_slice_buffer_add(dst, slice); break; diff --git a/src/core/lib/surface/byte_buffer_reader.c b/src/core/lib/surface/byte_buffer_reader.c index 1a6ccdaddb6..539b0142783 100644 --- a/src/core/lib/surface/byte_buffer_reader.c +++ b/src/core/lib/surface/byte_buffer_reader.c @@ -124,7 +124,7 @@ grpc_slice grpc_byte_buffer_reader_readall(grpc_byte_buffer_reader *reader) { grpc_slice in_slice; size_t bytes_read = 0; const size_t input_size = grpc_byte_buffer_length(reader->buffer_out); - grpc_slice out_slice = grpc_slice_malloc(input_size); + grpc_slice out_slice = GRPC_SLICE_MALLOC(input_size); uint8_t *const outbuf = GRPC_SLICE_START_PTR(out_slice); /* just an alias */ grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; From 4d780a52b8aa36397bb94f566ffb478d33be85d5 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 13:47:18 -0700 Subject: [PATCH 30/60] stuff --- build.yaml | 6 +++--- gRPC-Core.podspec | 6 ++++++ grpc.gemspec | 3 +++ package.xml | 3 +++ tools/doxygen/Doxyfile.core.internal | 3 +++ tools/run_tests/generated/sources_and_headers.json | 6 ++++++ vsprojects/vcxproj/gpr/gpr.vcxproj | 3 +++ vsprojects/vcxproj/gpr/gpr.vcxproj.filters | 9 +++++++++ 8 files changed, 36 insertions(+), 3 deletions(-) diff --git a/build.yaml b/build.yaml index cf22f19a988..6fa285b5466 100644 --- a/build.yaml +++ b/build.yaml @@ -86,13 +86,13 @@ filegroups: headers: - src/core/lib/profiling/timers.h - src/core/lib/support/arena.h + - src/core/lib/support/atomic.h + - src/core/lib/support/atomic_with_atm.h + - src/core/lib/support/atomic_with_std.h - src/core/lib/support/backoff.h - src/core/lib/support/block_annotate.h - src/core/lib/support/env.h - src/core/lib/support/memory.h - - src/core/lib/support/atomic.h - - src/core/lib/support/atomic_with_atm.h - - src/core/lib/support/atomic_with_std.h - src/core/lib/support/mpscq.h - src/core/lib/support/murmur_hash.h - src/core/lib/support/spinlock.h diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index db6a4974c78..5d9063d9ece 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -197,6 +197,9 @@ Pod::Spec.new do |s| # To save you from scrolling, this is the last part of the podspec. ss.source_files = 'src/core/lib/profiling/timers.h', 'src/core/lib/support/arena.h', + 'src/core/lib/support/atomic.h', + 'src/core/lib/support/atomic_with_atm.h', + 'src/core/lib/support/atomic_with_std.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', @@ -696,6 +699,9 @@ Pod::Spec.new do |s| ss.private_header_files = 'src/core/lib/profiling/timers.h', 'src/core/lib/support/arena.h', + 'src/core/lib/support/atomic.h', + 'src/core/lib/support/atomic_with_atm.h', + 'src/core/lib/support/atomic_with_std.h', 'src/core/lib/support/backoff.h', 'src/core/lib/support/block_annotate.h', 'src/core/lib/support/env.h', diff --git a/grpc.gemspec b/grpc.gemspec index cb6c51f898e..0889f2f0c2b 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -82,6 +82,9 @@ Gem::Specification.new do |s| s.files += %w( include/grpc/impl/codegen/sync_windows.h ) s.files += %w( src/core/lib/profiling/timers.h ) s.files += %w( src/core/lib/support/arena.h ) + s.files += %w( src/core/lib/support/atomic.h ) + s.files += %w( src/core/lib/support/atomic_with_atm.h ) + s.files += %w( src/core/lib/support/atomic_with_std.h ) s.files += %w( src/core/lib/support/backoff.h ) s.files += %w( src/core/lib/support/block_annotate.h ) s.files += %w( src/core/lib/support/env.h ) diff --git a/package.xml b/package.xml index 247e4ee76a8..6e601b4d984 100644 --- a/package.xml +++ b/package.xml @@ -91,6 +91,9 @@ + + + diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index eb45ba7c0ad..e5cbfb7560d 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1246,6 +1246,9 @@ src/core/lib/support/alloc.c \ src/core/lib/support/arena.c \ src/core/lib/support/arena.h \ src/core/lib/support/atm.c \ +src/core/lib/support/atomic.h \ +src/core/lib/support/atomic_with_atm.h \ +src/core/lib/support/atomic_with_std.h \ src/core/lib/support/avl.c \ src/core/lib/support/backoff.c \ src/core/lib/support/backoff.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 427976c797c..2de4acb2ffe 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7397,6 +7397,9 @@ "include/grpc/support/useful.h", "src/core/lib/profiling/timers.h", "src/core/lib/support/arena.h", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/backoff.h", "src/core/lib/support/block_annotate.h", "src/core/lib/support/env.h", @@ -7448,6 +7451,9 @@ "src/core/lib/support/arena.c", "src/core/lib/support/arena.h", "src/core/lib/support/atm.c", + "src/core/lib/support/atomic.h", + "src/core/lib/support/atomic_with_atm.h", + "src/core/lib/support/atomic_with_std.h", "src/core/lib/support/avl.c", "src/core/lib/support/backoff.c", "src/core/lib/support/backoff.h", diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj b/vsprojects/vcxproj/gpr/gpr.vcxproj index 474f4d25c9f..7fb81a7fbca 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj @@ -188,6 +188,9 @@ + + + diff --git a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters index 760de3070d7..27d9d2f38f4 100644 --- a/vsprojects/vcxproj/gpr/gpr.vcxproj.filters +++ b/vsprojects/vcxproj/gpr/gpr.vcxproj.filters @@ -260,6 +260,15 @@ src\core\lib\support + + src\core\lib\support + + + src\core\lib\support + + + src\core\lib\support + src\core\lib\support From ed88abcad8f2857701d2a14f21c92a9e9d4dbc0d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 13:52:16 -0700 Subject: [PATCH 31/60] Fix sanity --- src/core/lib/support/atomic.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/lib/support/atomic.h b/src/core/lib/support/atomic.h index 7cb495a0202..2226189b681 100644 --- a/src/core/lib/support/atomic.h +++ b/src/core/lib/support/atomic.h @@ -37,9 +37,9 @@ #include #ifdef GPR_HAS_CXX11_ATOMIC -#include "atomic_with_std.h" +#include "src/core/lib/support/atomic_with_std.h" #else -#include "atomic_with_atm.h" +#include "src/core/lib/support/atomic_with_atm.h" #endif #endif /* GRPC_CORE_LIB_SUPPORT_ATOMIC_H */ From 255ea13aa381f905b88a0816a0a29ee0315d251f Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 12 Apr 2017 17:18:56 -0700 Subject: [PATCH 32/60] Reduce duplication --- include/grpc/slice.h | 7 +++-- src/core/lib/slice/slice.c | 47 ++++++++----------------------- src/core/lib/slice/slice_buffer.c | 43 ++++++++-------------------- 3 files changed, 27 insertions(+), 70 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index 4dce6da9155..fe6379c867d 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -124,8 +124,11 @@ GPRAPI grpc_slice grpc_slice_sub_no_ref(grpc_slice s, size_t begin, size_t end); Requires s intialized, split <= s.length */ GPRAPI grpc_slice grpc_slice_split_tail(grpc_slice *s, size_t split); -/* The same as grpc_slice_split_tail, but without altering refcounts */ -GPRAPI grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *s, size_t split); +/* The same as grpc_slice_split_tail, but with an option to skip altering + * refcounts (grpc_slice_split_tail_maybe_ref(..., true) is equivalent to + * grpc_slice_split_tail(...)) */ +GPRAPI grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *s, size_t split, + bool inc_refs); /* Splits s into two: modifies s to be s[split:s.length], and returns a new slice, sharing a refcount with s, that contains s[0:split]. diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 8896f1dfd3f..285218837ac 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -312,7 +312,8 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { return subset; } -grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { +grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *source, size_t split, + bool incref) { grpc_slice tail; if (source->refcount == NULL) { @@ -334,9 +335,13 @@ grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { tail_length); } else { /* Build the result */ - tail.refcount = source->refcount->sub_refcount; - /* Bump the refcount */ - tail.refcount->vtable->ref(tail.refcount); + if (incref) { + tail.refcount = source->refcount->sub_refcount; + /* Bump the refcount */ + tail.refcount->vtable->ref(tail.refcount); + } else { + tail.refcount = &noop_refcount; + } /* Point into the source array */ tail.data.refcounted.bytes = source->data.refcounted.bytes + split; tail.data.refcounted.length = tail_length; @@ -348,38 +353,8 @@ grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { return tail; } -grpc_slice grpc_slice_split_tail_no_ref(grpc_slice *source, size_t split) { - grpc_slice tail; - - if (source->refcount == NULL) { - /* inlined data, copy it out */ - GPR_ASSERT(source->data.inlined.length >= split); - tail.refcount = NULL; - tail.data.inlined.length = (uint8_t)(source->data.inlined.length - split); - memcpy(tail.data.inlined.bytes, source->data.inlined.bytes + split, - tail.data.inlined.length); - source->data.inlined.length = (uint8_t)split; - } else { - size_t tail_length = source->data.refcounted.length - split; - GPR_ASSERT(source->data.refcounted.length >= split); - if (tail_length < sizeof(tail.data.inlined.bytes)) { - /* Copy out the bytes - it'll be cheaper than refcounting */ - tail.refcount = NULL; - tail.data.inlined.length = (uint8_t)tail_length; - memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split, - tail_length); - } else { - /* Build the result */ - tail.refcount = &noop_refcount; - /* Point into the source array */ - tail.data.refcounted.bytes = source->data.refcounted.bytes + split; - tail.data.refcounted.length = tail_length; - } - source->refcount = source->refcount->sub_refcount; - source->data.refcounted.length = split; - } - - return tail; +grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { + return grpc_slice_split_tail_maybe_ref(source, split, true); } grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index a13941e5ac8..5a4b434d28c 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -253,8 +253,9 @@ void grpc_slice_buffer_move_into(grpc_slice_buffer *src, src->length = 0; } -void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, - grpc_slice_buffer *dst) { +static void slice_buffer_move_first_maybe_ref(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst, + bool incref) { GPR_ASSERT(src->length >= n); if (src->length == n) { grpc_slice_buffer_move_into(src, dst); @@ -274,7 +275,8 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, grpc_slice_buffer_add(dst, slice); break; } else { /* n < slice_len */ - grpc_slice_buffer_undo_take_first(src, grpc_slice_split_tail(&slice, n)); + grpc_slice_buffer_undo_take_first( + src, grpc_slice_split_tail_maybe_ref(&slice, n, incref)); GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); grpc_slice_buffer_add(dst, slice); break; @@ -285,37 +287,14 @@ void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, GPR_ASSERT(src->count > 0); } +void grpc_slice_buffer_move_first(grpc_slice_buffer *src, size_t n, + grpc_slice_buffer *dst) { + slice_buffer_move_first_maybe_ref(src, n, dst, true); +} + void grpc_slice_buffer_move_first_no_ref(grpc_slice_buffer *src, size_t n, grpc_slice_buffer *dst) { - GPR_ASSERT(src->length >= n); - if (src->length == n) { - grpc_slice_buffer_move_into(src, dst); - return; - } - - size_t output_len = dst->length + n; - size_t new_input_len = src->length - n; - - while (src->count > 0) { - grpc_slice slice = grpc_slice_buffer_take_first(src); - size_t slice_len = GRPC_SLICE_LENGTH(slice); - if (n > slice_len) { - grpc_slice_buffer_add(dst, slice); - n -= slice_len; - } else if (n == slice_len) { - grpc_slice_buffer_add(dst, slice); - break; - } else { /* n < slice_len */ - grpc_slice_buffer_undo_take_first( - src, grpc_slice_split_tail_no_ref(&slice, n)); - GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); - grpc_slice_buffer_add(dst, slice); - break; - } - } - GPR_ASSERT(dst->length == output_len); - GPR_ASSERT(src->length == new_input_len); - GPR_ASSERT(src->count > 0); + slice_buffer_move_first_maybe_ref(src, n, dst, false); } void grpc_slice_buffer_move_first_into_buffer(grpc_exec_ctx *exec_ctx, From a3583b22eaf5448daff914a786d1e5214fe9aaec Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 13 Apr 2017 06:42:47 -0700 Subject: [PATCH 33/60] Fix compilation, add a check for no slow usages of grpc_slice_malloc --- include/grpc/slice.h | 2 +- src/core/lib/slice/slice.c | 2 +- tools/run_tests/sanity/core_banned_functions.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index fe6379c867d..86a455b42c2 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -128,7 +128,7 @@ GPRAPI grpc_slice grpc_slice_split_tail(grpc_slice *s, size_t split); * refcounts (grpc_slice_split_tail_maybe_ref(..., true) is equivalent to * grpc_slice_split_tail(...)) */ GPRAPI grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *s, size_t split, - bool inc_refs); + int inc_refs); /* Splits s into two: modifies s to be s[split:s.length], and returns a new slice, sharing a refcount with s, that contains s[0:split]. diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 285218837ac..e3d030235c8 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -313,7 +313,7 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { } grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *source, size_t split, - bool incref) { + int incref) { grpc_slice tail; if (source->refcount == NULL) { diff --git a/tools/run_tests/sanity/core_banned_functions.py b/tools/run_tests/sanity/core_banned_functions.py index c3c3cbec767..2387c5f1da0 100755 --- a/tools/run_tests/sanity/core_banned_functions.py +++ b/tools/run_tests/sanity/core_banned_functions.py @@ -50,6 +50,7 @@ BANNED_EXCEPT = { 'grpc_os_error(': ['src/core/lib/iomgr/error.c'], 'grpc_wsa_error(': ['src/core/lib/iomgr/error.c'], 'grpc_log_if_error(': ['src/core/lib/iomgr/error.c'], + 'grpc_slice_malloc(': ['src/core/lib/slice/slice.c'], } errors = 0 From 571c75aa330663e51e9eed3ad5b9f9d3898f3dcc Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 13 Apr 2017 16:39:37 -0700 Subject: [PATCH 34/60] Switch to Makefile based build on mac --- PYTHON-MANIFEST.in | 1 + setup.py | 18 ++++++++++++++---- src/python/grpcio/commands.py | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 846530532df..65876342198 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -17,3 +17,4 @@ include src/python/grpcio/support.py include src/python/grpcio/README.rst include requirements.txt include etc/roots.pem +include Makefile diff --git a/setup.py b/setup.py index 6ea79fc20a7..df97731ca02 100644 --- a/setup.py +++ b/setup.py @@ -102,7 +102,7 @@ ENABLE_DOCUMENTATION_BUILD = os.environ.get( EXTRA_ENV_COMPILE_ARGS = os.environ.get('GRPC_PYTHON_CFLAGS', None) EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) if EXTRA_ENV_COMPILE_ARGS is None: - EXTRA_ENV_COMPILE_ARGS = '-std=c++11' + EXTRA_ENV_COMPILE_ARGS = '' if 'win32' in sys.platform and sys.version_info < (3, 5): # We use define flags here and don't directly add to DEFINE_MACROS below to # ensure that the expert user/builder has a way of turning it off (via the @@ -114,7 +114,9 @@ if EXTRA_ENV_COMPILE_ARGS is None: EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' elif 'win32' in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC' - elif "linux" in sys.platform or "darwin" in sys.platform: + elif "linux" in sys.platform: + EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -fvisibility=hidden -fno-wrapv' + elif "darwin" in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -fvisibility=hidden -fno-wrapv' if EXTRA_ENV_LINK_ARGS is None: @@ -172,7 +174,7 @@ LDFLAGS = tuple(EXTRA_LINK_ARGS) CFLAGS = tuple(EXTRA_COMPILE_ARGS) if "linux" in sys.platform or "darwin" in sys.platform: pymodinit_type = 'PyObject*' if PY3 else 'void' - pymodinit = 'extern "C" __attribute__((visibility ("default"))) {}'.format(pymodinit_type) + pymodinit = '__attribute__((visibility ("default"))) {}'.format(pymodinit_type) DEFINE_MACROS += (('PyMODINIT_FUNC', pymodinit),) # By default, Python3 distutils enforces compatibility of @@ -192,14 +194,22 @@ def cython_extensions_and_necessity(): cython_module_files = [os.path.join(PYTHON_STEM, name.replace('.', '/') + '.pyx') for name in CYTHON_EXTENSION_MODULE_NAMES] + if "darwin" in sys.platform: + extra_objects = ['libs/opt/libares.a', + 'libs/opt/libboringssl.a', + 'libs/opt/libgpr.a', + 'libs/opt/libgrpc.a'] + CORE_C_FILES = [] + else: + extra_objects = [] extensions = [ _extension.Extension( name=module_name, - language='c++', sources=[module_file] + list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES), include_dirs=list(EXTENSION_INCLUDE_DIRECTORIES), libraries=list(EXTENSION_LIBRARIES), define_macros=list(DEFINE_MACROS), + extra_objects=extra_objects, extra_compile_args=list(CFLAGS), extra_link_args=list(LDFLAGS), ) for (module_name, module_file) in zip(list(CYTHON_EXTENSION_MODULE_NAMES), cython_module_files) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index f196b028a76..f6a66f1f418 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -266,6 +266,25 @@ class BuildExt(build_ext.build_ext): LINK_OPTIONS = {} def build_extensions(self): + if "darwin" in sys.platform: + target_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), + '..', '..', '..', 'libs', 'opt')) + targets = [os.path.join(target_path, 'libboringssl.a'), + os.path.join(target_path, 'libares.a'), + os.path.join(target_path, 'libgpr.a'), + os.path.join(target_path, 'libgrpc.a')] + make_process = subprocess.Popen(['make'] + targets, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + make_out, make_err = make_process.communicate() + if make_out and make_process.returncode != 0: + sys.stdout.write(make_out + '\n') + if make_err: + sys.stderr.write(make_err + '\n') + if make_process.returncode != 0: + raise Exception("make command failed!") + compiler = self.compiler.compiler_type if compiler in BuildExt.C_OPTIONS: for extension in self.extensions: From 2d9c9d940efbc19e3be8d37da54155109d9b1aa5 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 13 Apr 2017 17:07:32 -0700 Subject: [PATCH 35/60] linux fix --- setup.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index df97731ca02..047f0f1f5c3 100644 --- a/setup.py +++ b/setup.py @@ -199,13 +199,14 @@ def cython_extensions_and_necessity(): 'libs/opt/libboringssl.a', 'libs/opt/libgpr.a', 'libs/opt/libgrpc.a'] - CORE_C_FILES = [] + core_c_files = [] else: + core_c_files = list(CORE_C_FILES) extra_objects = [] extensions = [ _extension.Extension( name=module_name, - sources=[module_file] + list(CYTHON_HELPER_C_FILES) + list(CORE_C_FILES), + sources=[module_file] + list(CYTHON_HELPER_C_FILES) + core_c_files, include_dirs=list(EXTENSION_INCLUDE_DIRECTORIES), libraries=list(EXTENSION_LIBRARIES), define_macros=list(DEFINE_MACROS), From 1f24b2ad8a5b56db4ec2fa6e187ead604194d1c1 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Thu, 13 Apr 2017 17:34:52 -0700 Subject: [PATCH 36/60] Changes --- PYTHON-MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 65876342198..8be0af964f0 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -7,7 +7,7 @@ graft include/grpc graft third_party/boringssl graft third_party/nanopb graft third_party/zlib -graft third_party/c-ares +graft third_party/cares include src/python/grpcio/_spawn_patch.py include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py From 100539f7c8780053c07c21c22a67f08e86fd7f0e Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 15:20:34 -0700 Subject: [PATCH 37/60] No C99 in public headers --- include/grpc/impl/codegen/compression_types.h | 5 ++--- src/core/lib/iomgr/socket_mutator.h | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/grpc/impl/codegen/compression_types.h b/include/grpc/impl/codegen/compression_types.h index 170d99f431b..21932e9325f 100644 --- a/include/grpc/impl/codegen/compression_types.h +++ b/include/grpc/impl/codegen/compression_types.h @@ -34,7 +34,6 @@ #ifndef GRPC_IMPL_CODEGEN_COMPRESSION_TYPES_H #define GRPC_IMPL_CODEGEN_COMPRESSION_TYPES_H -#include #include #ifdef __cplusplus @@ -101,7 +100,7 @@ typedef struct grpc_compression_options { * precedence over \a default_algorithm. * TODO(dgq): currently only available for server channels. */ struct { - bool is_set; + int is_set; grpc_compression_level level; } default_level; @@ -109,7 +108,7 @@ typedef struct grpc_compression_options { * call specific settings. This option corresponds to the channel argument key * behind \a GRPC_COMPRESSION_CHANNEL_DEFAULT_ALGORITHM. */ struct { - bool is_set; + int is_set; grpc_compression_algorithm algorithm; } default_algorithm; diff --git a/src/core/lib/iomgr/socket_mutator.h b/src/core/lib/iomgr/socket_mutator.h index 2f5b6c248e9..28b1710ec40 100644 --- a/src/core/lib/iomgr/socket_mutator.h +++ b/src/core/lib/iomgr/socket_mutator.h @@ -37,6 +37,8 @@ #include #include +#include + #ifdef __cplusplus extern "C" { #endif From bc7c8457a25f41d4da37dc1d73c56070647ff19b Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 14 Apr 2017 15:22:24 -0700 Subject: [PATCH 38/60] No C99 in public headers --- include/grpc/impl/codegen/compression_types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/grpc/impl/codegen/compression_types.h b/include/grpc/impl/codegen/compression_types.h index 21932e9325f..a563711e326 100644 --- a/include/grpc/impl/codegen/compression_types.h +++ b/include/grpc/impl/codegen/compression_types.h @@ -34,7 +34,7 @@ #ifndef GRPC_IMPL_CODEGEN_COMPRESSION_TYPES_H #define GRPC_IMPL_CODEGEN_COMPRESSION_TYPES_H -#include +#include #ifdef __cplusplus extern "C" { From 5b034e6a4f574d9bc2d6f2bfb1f3d3a75785bc43 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Fri, 14 Apr 2017 16:25:51 -0700 Subject: [PATCH 39/60] Windows + mac dbg fix --- setup.py | 14 +++++++++----- src/python/grpcio/commands.py | 5 ++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 047f0f1f5c3..412e072e5b6 100644 --- a/setup.py +++ b/setup.py @@ -113,7 +113,7 @@ if EXTRA_ENV_COMPILE_ARGS is None: else: EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' elif 'win32' in sys.platform: - EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC' + EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC -std=c++11' elif "linux" in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -fvisibility=hidden -fno-wrapv' elif "darwin" in sys.platform: @@ -194,11 +194,15 @@ def cython_extensions_and_necessity(): cython_module_files = [os.path.join(PYTHON_STEM, name.replace('.', '/') + '.pyx') for name in CYTHON_EXTENSION_MODULE_NAMES] + config = os.environ['CONFIG'] + if config is None: + config = 'opt' + prefix = 'libs/' + config + '/' if "darwin" in sys.platform: - extra_objects = ['libs/opt/libares.a', - 'libs/opt/libboringssl.a', - 'libs/opt/libgpr.a', - 'libs/opt/libgrpc.a'] + extra_objects = [prefix + 'libares.a', + prefix + 'libboringssl.a', + prefix + 'libgpr.a', + prefix + 'libgrpc.a'] core_c_files = [] else: core_c_files = list(CORE_C_FILES) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index f6a66f1f418..7db5de63b2e 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -267,9 +267,12 @@ class BuildExt(build_ext.build_ext): def build_extensions(self): if "darwin" in sys.platform: + config = os.environ['CONFIG'] + if config is None: + config = 'opt' target_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), - '..', '..', '..', 'libs', 'opt')) + '..', '..', '..', 'libs', config)) targets = [os.path.join(target_path, 'libboringssl.a'), os.path.join(target_path, 'libares.a'), os.path.join(target_path, 'libgpr.a'), From ae5ca1b44770dd8cb0df74e91915e71116fa75ab Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Fri, 14 Apr 2017 16:34:26 -0700 Subject: [PATCH 40/60] Fixes --- setup.py | 4 +--- src/python/grpcio/commands.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 412e072e5b6..65c0a94a347 100644 --- a/setup.py +++ b/setup.py @@ -194,9 +194,7 @@ def cython_extensions_and_necessity(): cython_module_files = [os.path.join(PYTHON_STEM, name.replace('.', '/') + '.pyx') for name in CYTHON_EXTENSION_MODULE_NAMES] - config = os.environ['CONFIG'] - if config is None: - config = 'opt' + config = os.environ.get('CONFIG', 'opt') prefix = 'libs/' + config + '/' if "darwin" in sys.platform: extra_objects = [prefix + 'libares.a', diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 7db5de63b2e..848b284f354 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -267,9 +267,7 @@ class BuildExt(build_ext.build_ext): def build_extensions(self): if "darwin" in sys.platform: - config = os.environ['CONFIG'] - if config is None: - config = 'opt' + config = os.environ.get('CONFIG', opt) target_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', '..', 'libs', config)) From 63b0d11ca81b348dcf4463ff935ef326b4be7358 Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Mon, 17 Apr 2017 00:03:40 -0700 Subject: [PATCH 41/60] Final changes --- setup.py | 3 ++- src/python/grpcio/commands.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 65c0a94a347..4cbd1a45a90 100644 --- a/setup.py +++ b/setup.py @@ -104,6 +104,7 @@ EXTRA_ENV_LINK_ARGS = os.environ.get('GRPC_PYTHON_LDFLAGS', None) if EXTRA_ENV_COMPILE_ARGS is None: EXTRA_ENV_COMPILE_ARGS = '' if 'win32' in sys.platform and sys.version_info < (3, 5): + EXTRA_ENV_COMPILE_ARGS += ' -std=c++11' # We use define flags here and don't directly add to DEFINE_MACROS below to # ensure that the expert user/builder has a way of turning it off (via the # envvars) without adding yet more GRPC-specific envvars. @@ -113,7 +114,7 @@ if EXTRA_ENV_COMPILE_ARGS is None: else: EXTRA_ENV_COMPILE_ARGS += ' -D_ftime=_ftime64 -D_timeb=__timeb64' elif 'win32' in sys.platform: - EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC -std=c++11' + EXTRA_ENV_COMPILE_ARGS += ' -D_PYTHON_MSVC' elif "linux" in sys.platform: EXTRA_ENV_COMPILE_ARGS += ' -std=c++11 -fvisibility=hidden -fno-wrapv' elif "darwin" in sys.platform: diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 848b284f354..fa5620d7e97 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -267,7 +267,7 @@ class BuildExt(build_ext.build_ext): def build_extensions(self): if "darwin" in sys.platform: - config = os.environ.get('CONFIG', opt) + config = os.environ.get('CONFIG', 'opt') target_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', '..', 'libs', config)) From 0eac8aa2201a3500889059704727fbb7bd5cdcdd Mon Sep 17 00:00:00 2001 From: Ken Payson Date: Mon, 17 Apr 2017 01:58:15 -0700 Subject: [PATCH 42/60] Yapf_code --- src/python/grpcio/commands.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index fa5620d7e97..4f072809c47 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -269,22 +269,26 @@ class BuildExt(build_ext.build_ext): if "darwin" in sys.platform: config = os.environ.get('CONFIG', 'opt') target_path = os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), - '..', '..', '..', 'libs', config)) - targets = [os.path.join(target_path, 'libboringssl.a'), - os.path.join(target_path, 'libares.a'), - os.path.join(target_path, 'libgpr.a'), - os.path.join(target_path, 'libgrpc.a')] - make_process = subprocess.Popen(['make'] + targets, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + os.path.join( + os.path.dirname(os.path.realpath(__file__)), '..', '..', + '..', 'libs', config)) + targets = [ + os.path.join(target_path, 'libboringssl.a'), + os.path.join(target_path, 'libares.a'), + os.path.join(target_path, 'libgpr.a'), + os.path.join(target_path, 'libgrpc.a') + ] + make_process = subprocess.Popen( + ['make'] + targets, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) make_out, make_err = make_process.communicate() if make_out and make_process.returncode != 0: - sys.stdout.write(make_out + '\n') + sys.stdout.write(make_out + '\n') if make_err: - sys.stderr.write(make_err + '\n') + sys.stderr.write(make_err + '\n') if make_process.returncode != 0: - raise Exception("make command failed!") + raise Exception("make command failed!") compiler = self.compiler.compiler_type if compiler in BuildExt.C_OPTIONS: From 75c2270daa03b9031df582f477c10fbb1024b3e4 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 17 Apr 2017 12:21:32 -0700 Subject: [PATCH 43/60] Remove errant use of stdint.h --- include/grpc/impl/codegen/gpr_types.h | 1 - include/grpc/impl/codegen/grpc_types.h | 3 ++- include/grpc/impl/codegen/slice.h | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/include/grpc/impl/codegen/gpr_types.h b/include/grpc/impl/codegen/gpr_types.h index ed9976f4296..34d8156b615 100644 --- a/include/grpc/impl/codegen/gpr_types.h +++ b/include/grpc/impl/codegen/gpr_types.h @@ -37,7 +37,6 @@ #include #include -#include #ifdef __cplusplus extern "C" { diff --git a/include/grpc/impl/codegen/grpc_types.h b/include/grpc/impl/codegen/grpc_types.h index 06b67015e34..0a9840e774d 100644 --- a/include/grpc/impl/codegen/grpc_types.h +++ b/include/grpc/impl/codegen/grpc_types.h @@ -34,6 +34,8 @@ #ifndef GRPC_IMPL_CODEGEN_GRPC_TYPES_H #define GRPC_IMPL_CODEGEN_GRPC_TYPES_H +#include + #include #include #include @@ -41,7 +43,6 @@ #include #include -#include #ifdef __cplusplus extern "C" { diff --git a/include/grpc/impl/codegen/slice.h b/include/grpc/impl/codegen/slice.h index 0b09a0bfd89..b89a3f7910a 100644 --- a/include/grpc/impl/codegen/slice.h +++ b/include/grpc/impl/codegen/slice.h @@ -34,8 +34,9 @@ #ifndef GRPC_IMPL_CODEGEN_SLICE_H #define GRPC_IMPL_CODEGEN_SLICE_H +#include + #include -#include #include #include From 0d23d8954f2ee9b809d21ec4e156b2347c8c340c Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Thu, 20 Apr 2017 14:46:37 -0700 Subject: [PATCH 44/60] Add flexibility on *which* slice gets reffed Use it to ensure that sb_move_first acts predictably --- include/grpc/slice.h | 8 +++++++- src/core/lib/slice/slice.c | 30 ++++++++++++++++++++---------- src/core/lib/slice/slice_buffer.c | 10 ++++++++-- 3 files changed, 35 insertions(+), 13 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index d65bde49ee6..039347f17e3 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -124,11 +124,17 @@ GPRAPI grpc_slice grpc_slice_sub_no_ref(grpc_slice s, size_t begin, size_t end); Requires s intialized, split <= s.length */ GPRAPI grpc_slice grpc_slice_split_tail(grpc_slice *s, size_t split); +typedef enum { + GRPC_SLICE_REF_TAIL = 1, + GRPC_SLICE_REF_HEAD = 2, + GRPC_SLICE_REF_BOTH = 1 + 2 +} grpc_slice_ref_whom; + /* The same as grpc_slice_split_tail, but with an option to skip altering * refcounts (grpc_slice_split_tail_maybe_ref(..., true) is equivalent to * grpc_slice_split_tail(...)) */ GPRAPI grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *s, size_t split, - int inc_refs); + grpc_slice_ref_whom ref_whom); /* Splits s into two: modifies s to be s[split:s.length], and returns a new slice, sharing a refcount with s, that contains s[0:split]. diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 219d1a4efcf..9e69b443fe1 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -314,7 +314,7 @@ grpc_slice grpc_slice_sub(grpc_slice source, size_t begin, size_t end) { } grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *source, size_t split, - int incref) { + grpc_slice_ref_whom ref_whom) { grpc_slice tail; if (source->refcount == NULL) { @@ -328,26 +328,36 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *source, size_t split, } else { size_t tail_length = source->data.refcounted.length - split; GPR_ASSERT(source->data.refcounted.length >= split); - if (tail_length < sizeof(tail.data.inlined.bytes)) { + if (tail_length < sizeof(tail.data.inlined.bytes) && + ref_whom != GRPC_SLICE_REF_TAIL) { /* Copy out the bytes - it'll be cheaper than refcounting */ tail.refcount = NULL; tail.data.inlined.length = (uint8_t)tail_length; memcpy(tail.data.inlined.bytes, source->data.refcounted.bytes + split, tail_length); + source->refcount = source->refcount->sub_refcount; } else { /* Build the result */ - if (incref) { - tail.refcount = source->refcount->sub_refcount; - /* Bump the refcount */ - tail.refcount->vtable->ref(tail.refcount); - } else { - tail.refcount = &noop_refcount; + switch (ref_whom) { + case GRPC_SLICE_REF_TAIL: + tail.refcount = source->refcount->sub_refcount; + source->refcount = &noop_refcount; + break; + case GRPC_SLICE_REF_HEAD: + tail.refcount = &noop_refcount; + source->refcount = source->refcount->sub_refcount; + break; + case GRPC_SLICE_REF_BOTH: + tail.refcount = source->refcount->sub_refcount; + source->refcount = source->refcount->sub_refcount; + /* Bump the refcount */ + tail.refcount->vtable->ref(tail.refcount); + break; } /* Point into the source array */ tail.data.refcounted.bytes = source->data.refcounted.bytes + split; tail.data.refcounted.length = tail_length; } - source->refcount = source->refcount->sub_refcount; source->data.refcounted.length = split; } @@ -355,7 +365,7 @@ grpc_slice grpc_slice_split_tail_maybe_ref(grpc_slice *source, size_t split, } grpc_slice grpc_slice_split_tail(grpc_slice *source, size_t split) { - return grpc_slice_split_tail_maybe_ref(source, split, true); + return grpc_slice_split_tail_maybe_ref(source, split, GRPC_SLICE_REF_BOTH); } grpc_slice grpc_slice_split_head(grpc_slice *source, size_t split) { diff --git a/src/core/lib/slice/slice_buffer.c b/src/core/lib/slice/slice_buffer.c index 5a4b434d28c..e8d41ca0f7d 100644 --- a/src/core/lib/slice/slice_buffer.c +++ b/src/core/lib/slice/slice_buffer.c @@ -274,12 +274,18 @@ static void slice_buffer_move_first_maybe_ref(grpc_slice_buffer *src, size_t n, } else if (n == slice_len) { grpc_slice_buffer_add(dst, slice); break; - } else { /* n < slice_len */ + } else if (incref) { /* n < slice_len */ grpc_slice_buffer_undo_take_first( - src, grpc_slice_split_tail_maybe_ref(&slice, n, incref)); + src, grpc_slice_split_tail_maybe_ref(&slice, n, GRPC_SLICE_REF_BOTH)); GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); grpc_slice_buffer_add(dst, slice); break; + } else { /* n < slice_len */ + grpc_slice_buffer_undo_take_first( + src, grpc_slice_split_tail_maybe_ref(&slice, n, GRPC_SLICE_REF_TAIL)); + GPR_ASSERT(GRPC_SLICE_LENGTH(slice) == n); + grpc_slice_buffer_add_indexed(dst, slice); + break; } } GPR_ASSERT(dst->length == output_len); From 85bf34a4a8d0b8299ddda1b1abca401d8f2dd4a6 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 21 Apr 2017 16:14:59 -0700 Subject: [PATCH 45/60] Copy slices for in-process transports: its no longer safe to just ref --- include/grpc/slice.h | 3 +++ src/core/lib/slice/slice.c | 7 +++++++ test/core/util/passthru_endpoint.c | 4 ++-- test/core/util/trickle_endpoint.c | 7 +++---- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/include/grpc/slice.h b/include/grpc/slice.h index 039347f17e3..9c4b158ae8d 100644 --- a/include/grpc/slice.h +++ b/include/grpc/slice.h @@ -53,6 +53,9 @@ GPRAPI grpc_slice grpc_slice_ref(grpc_slice s); where dest!=NULL , then (*dest)(start, len). Requires s initialized. */ GPRAPI void grpc_slice_unref(grpc_slice s); +/* Copy slice - create a new slice that contains the same data as s */ +GPRAPI grpc_slice grpc_slice_copy(grpc_slice s); + /* Create a slice pointing at some data. Calls malloc to allocate a refcount for the object, and arranges that destroy will be called with the pointer passed in at destruction. */ diff --git a/src/core/lib/slice/slice.c b/src/core/lib/slice/slice.c index 9e69b443fe1..b90738fd1aa 100644 --- a/src/core/lib/slice/slice.c +++ b/src/core/lib/slice/slice.c @@ -55,6 +55,13 @@ grpc_slice grpc_empty_slice(void) { return out; } +grpc_slice grpc_slice_copy(grpc_slice s) { + grpc_slice out = GRPC_SLICE_MALLOC(GRPC_SLICE_LENGTH(s)); + memcpy(GRPC_SLICE_START_PTR(out), GRPC_SLICE_START_PTR(s), + GRPC_SLICE_LENGTH(s)); + return out; +} + grpc_slice grpc_slice_ref_internal(grpc_slice slice) { if (slice.refcount) { slice.refcount->vtable->ref(slice.refcount); diff --git a/test/core/util/passthru_endpoint.c b/test/core/util/passthru_endpoint.c index 121567fc0d2..ad718c75d1f 100644 --- a/test/core/util/passthru_endpoint.c +++ b/test/core/util/passthru_endpoint.c @@ -102,13 +102,13 @@ static void me_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("Endpoint already shutdown"); } else if (m->on_read != NULL) { for (size_t i = 0; i < slices->count; i++) { - grpc_slice_buffer_add(m->on_read_out, grpc_slice_ref(slices->slices[i])); + grpc_slice_buffer_add(m->on_read_out, grpc_slice_copy(slices->slices[i])); } grpc_closure_sched(exec_ctx, m->on_read, GRPC_ERROR_NONE); m->on_read = NULL; } else { for (size_t i = 0; i < slices->count; i++) { - grpc_slice_buffer_add(&m->read_buffer, grpc_slice_ref(slices->slices[i])); + grpc_slice_buffer_add(&m->read_buffer, grpc_slice_copy(slices->slices[i])); } } gpr_mu_unlock(&m->parent->mu); diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c index 0848147158a..66f30c8e65d 100644 --- a/test/core/util/trickle_endpoint.c +++ b/test/core/util/trickle_endpoint.c @@ -66,14 +66,13 @@ static void te_read(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, static void te_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, grpc_slice_buffer *slices, grpc_closure *cb) { trickle_endpoint *te = (trickle_endpoint *)ep; - for (size_t i = 0; i < slices->count; i++) { - grpc_slice_ref_internal(slices->slices[i]); - } gpr_mu_lock(&te->mu); if (te->write_buffer.length == 0) { te->last_write = gpr_now(GPR_CLOCK_MONOTONIC); } - grpc_slice_buffer_addn(&te->write_buffer, slices->slices, slices->count); + for (size_t i = 0; i < slices->count; i++) { + grpc_slice_buffer_add(&te->write_buffer, grpc_slice_copy(slices->slices[i])); + } grpc_closure_sched(exec_ctx, cb, GRPC_ERROR_REF(te->error)); gpr_mu_unlock(&te->mu); } From 1dd609b3c1a3ebb74cd0410d425ace362c66c964 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 24 Apr 2017 12:56:26 +0200 Subject: [PATCH 46/60] fix C++ distribtests --- test/distrib/cpp/run_distrib_test.sh | 9 ++------ .../artifacts/distribtest_targets.py | 22 +++++++++++++------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/test/distrib/cpp/run_distrib_test.sh b/test/distrib/cpp/run_distrib_test.sh index 15fbf281074..4728e32f065 100755 --- a/test/distrib/cpp/run_distrib_test.sh +++ b/test/distrib/cpp/run_distrib_test.sh @@ -30,13 +30,8 @@ set -ex -git clone $EXTERNAL_GIT_ROOT -# clone gRPC submodules, use data from locally cloned submodules where possible -(cd ${EXTERNAL_GIT_ROOT} && git submodule foreach 'cd /var/local/git/grpc \ -&& git submodule update --init --reference ${EXTERNAL_GIT_ROOT}/${name} \ -${name}') - -cd grpc +# change to grpc repo root +cd $(dirname $0)/../../.. cd third_party/protobuf && ./autogen.sh && \ ./configure && make -j4 && make check && make install && ldconfig diff --git a/tools/run_tests/artifacts/distribtest_targets.py b/tools/run_tests/artifacts/distribtest_targets.py index 90bbde83cf4..097fd2d8b5e 100644 --- a/tools/run_tests/artifacts/distribtest_targets.py +++ b/tools/run_tests/artifacts/distribtest_targets.py @@ -38,11 +38,14 @@ import python_utils.jobset as jobset def create_docker_jobspec(name, dockerfile_dir, shell_command, environ={}, - flake_retries=0, timeout_retries=0): + flake_retries=0, timeout_retries=0, + copy_rel_path=None): """Creates jobspec for a task running under docker.""" environ = environ.copy() environ['RUN_COMMAND'] = shell_command - environ['RELATIVE_COPY_PATH'] = 'test/distrib' + # the entire repo will be cloned if copy_rel_path is not set. + if copy_rel_path: + environ['RELATIVE_COPY_PATH'] = copy_rel_path docker_args=[] for k,v in environ.items(): @@ -102,7 +105,8 @@ class CSharpDistribTest(object): 'tools/dockerfile/distribtest/csharp_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix) + 'test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix, + copy_rel_path='test/distrib') elif self.platform == 'macos': return create_jobspec(self.name, ['test/distrib/csharp/run_distrib_test%s.sh' % self.script_suffix], @@ -151,7 +155,8 @@ class NodeDistribTest(object): self.arch), '%s test/distrib/node/run_distrib_test.sh %s' % ( linux32, - self.node_version)) + self.node_version), + copy_rel_path='test/distrib') elif self.platform == 'macos': return create_jobspec(self.name, ['test/distrib/node/run_distrib_test.sh', @@ -185,7 +190,8 @@ class PythonDistribTest(object): 'tools/dockerfile/distribtest/python_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/python/run_distrib_test.sh') + 'test/distrib/python/run_distrib_test.sh', + copy_rel_path='test/distrib') def __str__(self): return self.name @@ -212,7 +218,8 @@ class RubyDistribTest(object): 'tools/dockerfile/distribtest/ruby_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/ruby/run_distrib_test.sh') + 'test/distrib/ruby/run_distrib_test.sh', + copy_rel_path='test/distrib') def __str__(self): return self.name @@ -237,7 +244,8 @@ class PHPDistribTest(object): 'tools/dockerfile/distribtest/php_%s_%s' % ( self.docker_suffix, self.arch), - 'test/distrib/php/run_distrib_test.sh') + 'test/distrib/php/run_distrib_test.sh', + copy_rel_path='test/distrib') elif self.platform == 'macos': return create_jobspec(self.name, ['test/distrib/php/run_distrib_test.sh'], From e0e961aae70a3764d211de8f07610585db63daf2 Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Mon, 24 Apr 2017 09:11:44 -0700 Subject: [PATCH 47/60] fix guard on src/core/tsi/transport_security_adapter.h --- memory_usage.csv | 1 + src/core/tsi/transport_security_adapter.h | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 memory_usage.csv diff --git a/memory_usage.csv b/memory_usage.csv new file mode 100644 index 00000000000..6eaf90f61fd --- /dev/null +++ b/memory_usage.csv @@ -0,0 +1 @@ +5678.136000,586484,4195801,5849.208667,205445,, diff --git a/src/core/tsi/transport_security_adapter.h b/src/core/tsi/transport_security_adapter.h index 686618b53b1..e4dc07515f1 100644 --- a/src/core/tsi/transport_security_adapter.h +++ b/src/core/tsi/transport_security_adapter.h @@ -31,8 +31,8 @@ * */ -#ifndef GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H -#define GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H +#ifndef GRPC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H +#define GRPC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H #include "src/core/tsi/transport_security_interface.h" @@ -59,4 +59,4 @@ tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter); } #endif -#endif // GRPC_SRC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H +#endif // GRPC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H From 1c2332b0c1edfdcd5746939feaabc0904475d7ea Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 24 Apr 2017 09:13:35 -0700 Subject: [PATCH 48/60] Fix potential memory leak, cleanup some code --- .../message_compress_filter.c | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/core/ext/filters/http/message_compress/message_compress_filter.c b/src/core/ext/filters/http/message_compress/message_compress_filter.c index 4f5f41e9b04..1da8cf69cbf 100644 --- a/src/core/ext/filters/http/message_compress/message_compress_filter.c +++ b/src/core/ext/filters/http/message_compress/message_compress_filter.c @@ -277,13 +277,10 @@ static void compress_start_transport_stream_op_batch( GPR_TIMER_BEGIN("compress_start_transport_stream_op_batch", 0); if (op->cancel_stream) { - gpr_atm cur; GRPC_ERROR_REF(op->payload->cancel_stream.cancel_error); - do { - cur = gpr_atm_acq_load(&calld->send_initial_metadata_state); - } while (!gpr_atm_rel_cas( - &calld->send_initial_metadata_state, cur, - CANCELLED_BIT | (gpr_atm)op->payload->cancel_stream.cancel_error)); + gpr_atm cur = gpr_atm_full_xchg( + &calld->send_initial_metadata_state, + CANCELLED_BIT | (gpr_atm)op->payload->cancel_stream.cancel_error); switch (cur) { case HAS_COMPRESSION_ALGORITHM: case NO_COMPRESSION_ALGORITHM: @@ -311,13 +308,18 @@ static void compress_start_transport_stream_op_batch( grpc_transport_stream_op_batch_finish_with_failure(exec_ctx, op, error); return; } - gpr_atm cur = gpr_atm_acq_load(&calld->send_initial_metadata_state); + gpr_atm cur; + retry_send_im: + cur = gpr_atm_acq_load(&calld->send_initial_metadata_state); GPR_ASSERT(cur != HAS_COMPRESSION_ALGORITHM && cur != NO_COMPRESSION_ALGORITHM); if ((cur & CANCELLED_BIT) == 0) { - gpr_atm_rel_store(&calld->send_initial_metadata_state, - has_compression_algorithm ? HAS_COMPRESSION_ALGORITHM - : NO_COMPRESSION_ALGORITHM); + if (!gpr_atm_rel_cas(&calld->send_initial_metadata_state, cur, + has_compression_algorithm + ? HAS_COMPRESSION_ALGORITHM + : NO_COMPRESSION_ALGORITHM)) { + goto retry_send_im; + } if (cur != INITIAL_METADATA_UNSEEN) { grpc_call_next_op(exec_ctx, elem, (grpc_transport_stream_op_batch *)cur); From f2263e14ca778e0a42d896ad5228c671fbb97aed Mon Sep 17 00:00:00 2001 From: jiangtaoli2016 Date: Mon, 24 Apr 2017 15:32:03 -0700 Subject: [PATCH 49/60] fix guards in transport_security_adapter.h --- memory_usage.csv | 1 - src/core/tsi/transport_security_adapter.h | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 memory_usage.csv diff --git a/memory_usage.csv b/memory_usage.csv deleted file mode 100644 index 6eaf90f61fd..00000000000 --- a/memory_usage.csv +++ /dev/null @@ -1 +0,0 @@ -5678.136000,586484,4195801,5849.208667,205445,, diff --git a/src/core/tsi/transport_security_adapter.h b/src/core/tsi/transport_security_adapter.h index e4dc07515f1..400df2f11bd 100644 --- a/src/core/tsi/transport_security_adapter.h +++ b/src/core/tsi/transport_security_adapter.h @@ -59,4 +59,4 @@ tsi_handshaker *tsi_adapter_handshaker_get_wrapped(tsi_handshaker *adapter); } #endif -#endif // GRPC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H +#endif /* GRPC_CORE_TSI_TRANSPORT_SECURITY_ADAPTER_H */ From 0f3b87aef244ca92527f34754ee74b3c1608f38f Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 26 Apr 2017 01:22:18 -0700 Subject: [PATCH 50/60] Fix C/C++ test filtering --- tools/jenkins/run_c_cpp_test.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/jenkins/run_c_cpp_test.sh b/tools/jenkins/run_c_cpp_test.sh index a7e574518e3..afa2e780f75 100755 --- a/tools/jenkins/run_c_cpp_test.sh +++ b/tools/jenkins/run_c_cpp_test.sh @@ -35,10 +35,12 @@ set -ex # Enter the gRPC repo root cd $(dirname $0)/../.. -AFFECTS_C_CPP=`python -c 'import sys; \ +AFFECTS_C_CPP=`python -c 'import os; \ + import sys; \ sys.path.insert(0, "tools/run_tests/python_utils"); \ import filter_pull_request_tests as filter; \ - print(filter.affects_c_cpp("origin/$ghprbTargetBranch"))'` + github_target_branch = os.environ.get("ghprbTargetBranch"); \ + print(filter.affects_c_cpp("origin/%s" % github_target_branch))'` if [ $AFFECTS_C_CPP == "False" ] ; then echo "This pull request does not affect C/C++. Tests do not need to be run." From e38948a1cdd44554095ad8b277e86877dfda1301 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 26 Apr 2017 08:53:25 -0700 Subject: [PATCH 51/60] clang-format --- test/core/util/passthru_endpoint.c | 3 ++- test/core/util/trickle_endpoint.c | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/core/util/passthru_endpoint.c b/test/core/util/passthru_endpoint.c index ad718c75d1f..6400845d235 100644 --- a/test/core/util/passthru_endpoint.c +++ b/test/core/util/passthru_endpoint.c @@ -108,7 +108,8 @@ static void me_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, m->on_read = NULL; } else { for (size_t i = 0; i < slices->count; i++) { - grpc_slice_buffer_add(&m->read_buffer, grpc_slice_copy(slices->slices[i])); + grpc_slice_buffer_add(&m->read_buffer, + grpc_slice_copy(slices->slices[i])); } } gpr_mu_unlock(&m->parent->mu); diff --git a/test/core/util/trickle_endpoint.c b/test/core/util/trickle_endpoint.c index 66f30c8e65d..58ac59711be 100644 --- a/test/core/util/trickle_endpoint.c +++ b/test/core/util/trickle_endpoint.c @@ -71,7 +71,8 @@ static void te_write(grpc_exec_ctx *exec_ctx, grpc_endpoint *ep, te->last_write = gpr_now(GPR_CLOCK_MONOTONIC); } for (size_t i = 0; i < slices->count; i++) { - grpc_slice_buffer_add(&te->write_buffer, grpc_slice_copy(slices->slices[i])); + grpc_slice_buffer_add(&te->write_buffer, + grpc_slice_copy(slices->slices[i])); } grpc_closure_sched(exec_ctx, cb, GRPC_ERROR_REF(te->error)); gpr_mu_unlock(&te->mu); From bb4046cd771344f69b3268a86c48b01cd34a921d Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Wed, 26 Apr 2017 08:56:30 -0700 Subject: [PATCH 52/60] Regenerate projects --- grpc.def | 1 + src/ruby/ext/grpc/rb_grpc_imports.generated.c | 2 ++ src/ruby/ext/grpc/rb_grpc_imports.generated.h | 5 ++++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/grpc.def b/grpc.def index 54f9609bc2a..8b64b37ff34 100644 --- a/grpc.def +++ b/grpc.def @@ -142,6 +142,7 @@ EXPORTS grpc_server_credentials_set_auth_metadata_processor grpc_slice_ref grpc_slice_unref + grpc_slice_copy grpc_slice_new grpc_slice_new_with_user_data grpc_slice_new_with_len diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index b6360f2d4c9..d11f7f6a78a 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -180,6 +180,7 @@ grpc_call_set_credentials_type grpc_call_set_credentials_import; grpc_server_credentials_set_auth_metadata_processor_type grpc_server_credentials_set_auth_metadata_processor_import; grpc_slice_ref_type grpc_slice_ref_import; grpc_slice_unref_type grpc_slice_unref_import; +grpc_slice_copy_type grpc_slice_copy_import; grpc_slice_new_type grpc_slice_new_import; grpc_slice_new_with_user_data_type grpc_slice_new_with_user_data_import; grpc_slice_new_with_len_type grpc_slice_new_with_len_import; @@ -481,6 +482,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_server_credentials_set_auth_metadata_processor_import = (grpc_server_credentials_set_auth_metadata_processor_type) GetProcAddress(library, "grpc_server_credentials_set_auth_metadata_processor"); grpc_slice_ref_import = (grpc_slice_ref_type) GetProcAddress(library, "grpc_slice_ref"); grpc_slice_unref_import = (grpc_slice_unref_type) GetProcAddress(library, "grpc_slice_unref"); + grpc_slice_copy_import = (grpc_slice_copy_type) GetProcAddress(library, "grpc_slice_copy"); grpc_slice_new_import = (grpc_slice_new_type) GetProcAddress(library, "grpc_slice_new"); grpc_slice_new_with_user_data_import = (grpc_slice_new_with_user_data_type) GetProcAddress(library, "grpc_slice_new_with_user_data"); grpc_slice_new_with_len_import = (grpc_slice_new_with_len_type) GetProcAddress(library, "grpc_slice_new_with_len"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 539b7de9bd3..dc50b87baf2 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -491,6 +491,9 @@ extern grpc_slice_ref_type grpc_slice_ref_import; typedef void(*grpc_slice_unref_type)(grpc_slice s); extern grpc_slice_unref_type grpc_slice_unref_import; #define grpc_slice_unref grpc_slice_unref_import +typedef grpc_slice(*grpc_slice_copy_type)(grpc_slice s); +extern grpc_slice_copy_type grpc_slice_copy_import; +#define grpc_slice_copy grpc_slice_copy_import typedef grpc_slice(*grpc_slice_new_type)(void *p, size_t len, void (*destroy)(void *)); extern grpc_slice_new_type grpc_slice_new_import; #define grpc_slice_new grpc_slice_new_import @@ -530,7 +533,7 @@ extern grpc_slice_sub_no_ref_type grpc_slice_sub_no_ref_import; typedef grpc_slice(*grpc_slice_split_tail_type)(grpc_slice *s, size_t split); extern grpc_slice_split_tail_type grpc_slice_split_tail_import; #define grpc_slice_split_tail grpc_slice_split_tail_import -typedef grpc_slice(*grpc_slice_split_tail_maybe_ref_type)(grpc_slice *s, size_t split, int inc_refs); +typedef grpc_slice(*grpc_slice_split_tail_maybe_ref_type)(grpc_slice *s, size_t split, grpc_slice_ref_whom ref_whom); extern grpc_slice_split_tail_maybe_ref_type grpc_slice_split_tail_maybe_ref_import; #define grpc_slice_split_tail_maybe_ref grpc_slice_split_tail_maybe_ref_import typedef grpc_slice(*grpc_slice_split_head_type)(grpc_slice *s, size_t split); From b71ef65cb379c9d44ca9dcaf03306196c7c04c4f Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Wed, 12 Apr 2017 21:44:49 -0700 Subject: [PATCH 53/60] more TLS cleanup. - using NULL-terminated strings as opposed to const unsigned char * with length since the credentials are in PEM format. - aligning the structures with gRPC so that we have less convertions to do back and forth. --- include/grpc/grpc_security.h | 2 +- .../lib/http/httpcli_security_connector.c | 16 +- .../credentials/ssl/ssl_credentials.c | 76 +++------ .../security/transport/security_connector.c | 70 ++++---- .../security/transport/security_connector.h | 24 ++- src/core/tsi/ssl_transport_security.c | 149 ++++++++---------- src/core/tsi/ssl_transport_security.h | 100 +++++------- 7 files changed, 176 insertions(+), 261 deletions(-) diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index 79199cc5d68..5d3cc4fd674 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -158,7 +158,7 @@ typedef struct { } grpc_ssl_pem_key_cert_pair; /* Creates an SSL credentials object. - - pem_roots_cert is the NULL-terminated string containing the PEM encoding + - pem_root_certs is the NULL-terminated string containing the PEM encoding of the server root certificates. If this parameter is NULL, the implementation will first try to dereference the file pointed by the GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment variable, and if that fails, diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c index 9eab1360a4a..76946434f06 100644 --- a/src/core/lib/http/httpcli_security_connector.c +++ b/src/core/lib/http/httpcli_security_connector.c @@ -106,9 +106,8 @@ static grpc_security_connector_vtable httpcli_ssl_vtable = { httpcli_ssl_destroy, httpcli_ssl_check_peer}; static grpc_security_status httpcli_ssl_channel_security_connector_create( - grpc_exec_ctx *exec_ctx, const unsigned char *pem_root_certs, - size_t pem_root_certs_size, const char *secure_peer_name, - grpc_channel_security_connector **sc) { + grpc_exec_ctx *exec_ctx, const char *pem_root_certs, + const char *secure_peer_name, grpc_channel_security_connector **sc) { tsi_result result = TSI_OK; grpc_httpcli_ssl_channel_security_connector *c; @@ -126,8 +125,7 @@ static grpc_security_status httpcli_ssl_channel_security_connector_create( c->secure_peer_name = gpr_strdup(secure_peer_name); } result = tsi_create_ssl_client_handshaker_factory( - NULL, 0, NULL, 0, pem_root_certs, pem_root_certs_size, NULL, NULL, NULL, - 0, &c->handshaker_factory); + NULL, pem_root_certs, NULL, NULL, 0, &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -173,10 +171,9 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, void (*on_done)(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *endpoint)) { grpc_channel_security_connector *sc = NULL; - const unsigned char *pem_root_certs = NULL; on_done_closure *c = gpr_malloc(sizeof(*c)); - size_t pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); - if (pem_root_certs == NULL || pem_root_certs_size == 0) { + const char *pem_root_certs = grpc_get_default_ssl_roots(); + if (pem_root_certs == NULL) { gpr_log(GPR_ERROR, "Could not get default pem root certs."); on_done(exec_ctx, arg, NULL); gpr_free(c); @@ -186,8 +183,7 @@ static void ssl_handshake(grpc_exec_ctx *exec_ctx, void *arg, c->arg = arg; c->handshake_mgr = grpc_handshake_manager_create(); GPR_ASSERT(httpcli_ssl_channel_security_connector_create( - exec_ctx, pem_root_certs, pem_root_certs_size, host, &sc) == - GRPC_SECURITY_OK); + exec_ctx, pem_root_certs, host, &sc) == GRPC_SECURITY_OK); grpc_channel_security_connector_add_handshakers(exec_ctx, sc, c->handshake_mgr); grpc_handshake_manager_do_handshake( diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c index b63bb6b6e9e..7c35ebe6841 100644 --- a/src/core/lib/security/credentials/ssl/ssl_credentials.c +++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c @@ -40,28 +40,24 @@ #include #include +#include // -// Utils +// SSL Channel Credentials. // -static void ssl_copy_key_material(const char *input, unsigned char **output, - size_t *output_size) { - *output_size = strlen(input); - *output = gpr_malloc(*output_size); - memcpy(*output, input, *output_size); +static void ssl_config_pem_key_cert_pair_destroy( + tsi_ssl_pem_key_cert_pair *kp) { + if (kp == NULL) return; + gpr_free((void *)kp->private_key); + gpr_free((void *)kp->cert_chain); } -// -// SSL Channel Credentials. -// - static void ssl_destruct(grpc_exec_ctx *exec_ctx, grpc_channel_credentials *creds) { grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds; - if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs); - if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key); - if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain); + gpr_free(c->config.pem_root_certs); + ssl_config_pem_key_cert_pair_destroy(&c->config.pem_key_cert_pair); } static grpc_security_status ssl_create_security_connector( @@ -102,18 +98,15 @@ static void ssl_build_config(const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair, grpc_ssl_config *config) { if (pem_root_certs != NULL) { - ssl_copy_key_material(pem_root_certs, &config->pem_root_certs, - &config->pem_root_certs_size); + config->pem_root_certs = gpr_strdup(pem_root_certs); } if (pem_key_cert_pair != NULL) { GPR_ASSERT(pem_key_cert_pair->private_key != NULL); GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL); - ssl_copy_key_material(pem_key_cert_pair->private_key, - &config->pem_private_key, - &config->pem_private_key_size); - ssl_copy_key_material(pem_key_cert_pair->cert_chain, - &config->pem_cert_chain, - &config->pem_cert_chain_size); + config->pem_key_cert_pair.cert_chain = + gpr_strdup(pem_key_cert_pair->cert_chain); + config->pem_key_cert_pair.private_key = + gpr_strdup(pem_key_cert_pair->private_key); } } @@ -143,22 +136,10 @@ static void ssl_server_destruct(grpc_exec_ctx *exec_ctx, grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds; size_t i; for (i = 0; i < c->config.num_key_cert_pairs; i++) { - if (c->config.pem_private_keys[i] != NULL) { - gpr_free(c->config.pem_private_keys[i]); - } - if (c->config.pem_cert_chains[i] != NULL) { - gpr_free(c->config.pem_cert_chains[i]); - } - } - if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys); - if (c->config.pem_private_keys_sizes != NULL) { - gpr_free(c->config.pem_private_keys_sizes); - } - if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains); - if (c->config.pem_cert_chains_sizes != NULL) { - gpr_free(c->config.pem_cert_chains_sizes); + ssl_config_pem_key_cert_pair_destroy(&c->config.pem_key_cert_pairs[i]); } - if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs); + gpr_free(c->config.pem_key_cert_pairs); + gpr_free(c->config.pem_root_certs); } static grpc_security_status ssl_server_create_security_connector( @@ -179,30 +160,21 @@ static void ssl_build_server_config( size_t i; config->client_certificate_request = client_certificate_request; if (pem_root_certs != NULL) { - ssl_copy_key_material(pem_root_certs, &config->pem_root_certs, - &config->pem_root_certs_size); + config->pem_root_certs = gpr_strdup(pem_root_certs); } if (num_key_cert_pairs > 0) { GPR_ASSERT(pem_key_cert_pairs != NULL); - config->pem_private_keys = - gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *)); - config->pem_cert_chains = - gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *)); - config->pem_private_keys_sizes = - gpr_malloc(num_key_cert_pairs * sizeof(size_t)); - config->pem_cert_chains_sizes = - gpr_malloc(num_key_cert_pairs * sizeof(size_t)); + config->pem_key_cert_pairs = + gpr_zalloc(num_key_cert_pairs * sizeof(tsi_ssl_pem_key_cert_pair)); } config->num_key_cert_pairs = num_key_cert_pairs; for (i = 0; i < num_key_cert_pairs; i++) { GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL); GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL); - ssl_copy_key_material(pem_key_cert_pairs[i].private_key, - &config->pem_private_keys[i], - &config->pem_private_keys_sizes[i]); - ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain, - &config->pem_cert_chains[i], - &config->pem_cert_chains_sizes[i]); + config->pem_key_cert_pairs[i].cert_chain = + gpr_strdup(pem_key_cert_pairs[i].cert_chain); + config->pem_key_cert_pairs[i].private_key = + gpr_strdup(pem_key_cert_pairs[i].private_key); } } diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index b15196e6770..a7a5d65caf6 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -695,6 +695,7 @@ static grpc_security_connector_vtable ssl_channel_vtable = { static grpc_security_connector_vtable ssl_server_vtable = { ssl_server_destroy, ssl_server_check_peer}; +/* returns a NULL terminated slice. */ static grpc_slice compute_default_pem_root_certs_once(void) { grpc_slice result = grpc_empty_slice(); @@ -703,7 +704,7 @@ static grpc_slice compute_default_pem_root_certs_once(void) { gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); if (default_root_certs_path != NULL) { GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(default_root_certs_path, 0, &result)); + grpc_load_file(default_root_certs_path, 1, &result)); gpr_free(default_root_certs_path); } @@ -714,15 +715,18 @@ static grpc_slice compute_default_pem_root_certs_once(void) { ovrd_res = ssl_roots_override_cb(&pem_root_certs); if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) { GPR_ASSERT(pem_root_certs != NULL); - result = grpc_slice_new(pem_root_certs, strlen(pem_root_certs), gpr_free); + result = grpc_slice_from_copied_buffer( + pem_root_certs, + strlen(pem_root_certs) + 1); // NULL terminator. } + gpr_free(pem_root_certs); } /* Fall back to installed certs if needed. */ if (GRPC_SLICE_IS_EMPTY(result) && ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { GRPC_LOG_IF_ERROR("load_file", - grpc_load_file(installed_roots_path, 0, &result)); + grpc_load_file(installed_roots_path, 1, &result)); } return result; } @@ -762,13 +766,14 @@ get_tsi_client_certificate_request_type( } } -size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) { +const char *grpc_get_default_ssl_roots(void) { /* TODO(jboeuf@google.com): Maybe revisit the approach which consists in loading all the roots once for the lifetime of the process. */ static gpr_once once = GPR_ONCE_INIT; gpr_once_init(&once, init_default_pem_root_certs); - *pem_root_certs = GRPC_SLICE_START_PTR(default_pem_root_certs); - return GRPC_SLICE_LENGTH(default_pem_root_certs); + return GRPC_SLICE_IS_EMPTY(default_pem_root_certs) + ? NULL + : (const char *)GRPC_SLICE_START_PTR(default_pem_root_certs); } grpc_security_status grpc_ssl_channel_security_connector_create( @@ -776,22 +781,16 @@ grpc_security_status grpc_ssl_channel_security_connector_create( const grpc_ssl_config *config, const char *target_name, const char *overridden_target_name, grpc_channel_security_connector **sc) { size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions(); - const unsigned char **alpn_protocol_strings = + const char **alpn_protocol_strings = gpr_malloc(sizeof(const char *) * num_alpn_protocols); - unsigned char *alpn_protocol_string_lengths = - gpr_malloc(sizeof(unsigned char) * num_alpn_protocols); tsi_result result = TSI_OK; grpc_ssl_channel_security_connector *c; size_t i; - const unsigned char *pem_root_certs; - size_t pem_root_certs_size; + const char *pem_root_certs; char *port; for (i = 0; i < num_alpn_protocols; i++) { - alpn_protocol_strings[i] = - (const unsigned char *)grpc_chttp2_get_alpn_version_index(i); - alpn_protocol_string_lengths[i] = - (unsigned char)strlen(grpc_chttp2_get_alpn_version_index(i)); + alpn_protocol_strings[i] = grpc_chttp2_get_alpn_version_index(i); } if (config == NULL || target_name == NULL) { @@ -799,14 +798,13 @@ grpc_security_status grpc_ssl_channel_security_connector_create( goto error; } if (config->pem_root_certs == NULL) { - pem_root_certs_size = grpc_get_default_ssl_roots(&pem_root_certs); - if (pem_root_certs == NULL || pem_root_certs_size == 0) { + pem_root_certs = grpc_get_default_ssl_roots(); + if (pem_root_certs == NULL) { gpr_log(GPR_ERROR, "Could not get default pem root certs."); goto error; } } else { pem_root_certs = config->pem_root_certs; - pem_root_certs_size = config->pem_root_certs_size; } c = gpr_zalloc(sizeof(grpc_ssl_channel_security_connector)); @@ -823,11 +821,12 @@ grpc_security_status grpc_ssl_channel_security_connector_create( if (overridden_target_name != NULL) { c->overridden_target_name = gpr_strdup(overridden_target_name); } + + bool has_key_cert_pair = config->pem_key_cert_pair.private_key != NULL && + config->pem_key_cert_pair.cert_chain != NULL; result = tsi_create_ssl_client_handshaker_factory( - config->pem_private_key, config->pem_private_key_size, - config->pem_cert_chain, config->pem_cert_chain_size, pem_root_certs, - pem_root_certs_size, ssl_cipher_suites(), alpn_protocol_strings, - alpn_protocol_string_lengths, (uint16_t)num_alpn_protocols, + has_key_cert_pair ? &config->pem_key_cert_pair : NULL, pem_root_certs, + ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", @@ -838,12 +837,10 @@ grpc_security_status grpc_ssl_channel_security_connector_create( } *sc = &c->base; gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_OK; error: gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_ERROR; } @@ -851,19 +848,14 @@ grpc_security_status grpc_ssl_server_security_connector_create( grpc_exec_ctx *exec_ctx, const grpc_ssl_server_config *config, grpc_server_security_connector **sc) { size_t num_alpn_protocols = grpc_chttp2_num_alpn_versions(); - const unsigned char **alpn_protocol_strings = + const char **alpn_protocol_strings = gpr_malloc(sizeof(const char *) * num_alpn_protocols); - unsigned char *alpn_protocol_string_lengths = - gpr_malloc(sizeof(unsigned char) * num_alpn_protocols); tsi_result result = TSI_OK; grpc_ssl_server_security_connector *c; size_t i; for (i = 0; i < num_alpn_protocols; i++) { - alpn_protocol_strings[i] = - (const unsigned char *)grpc_chttp2_get_alpn_version_index(i); - alpn_protocol_string_lengths[i] = - (unsigned char)strlen(grpc_chttp2_get_alpn_version_index(i)); + alpn_protocol_strings[i] = grpc_chttp2_get_alpn_version_index(i); } if (config == NULL || config->num_key_cert_pairs == 0) { @@ -876,15 +868,11 @@ grpc_security_status grpc_ssl_server_security_connector_create( c->base.base.url_scheme = GRPC_SSL_URL_SCHEME; c->base.base.vtable = &ssl_server_vtable; result = tsi_create_ssl_server_handshaker_factory_ex( - (const unsigned char **)config->pem_private_keys, - config->pem_private_keys_sizes, - (const unsigned char **)config->pem_cert_chains, - config->pem_cert_chains_sizes, config->num_key_cert_pairs, - config->pem_root_certs, config->pem_root_certs_size, - get_tsi_client_certificate_request_type( - config->client_certificate_request), - ssl_cipher_suites(), alpn_protocol_strings, alpn_protocol_string_lengths, - (uint16_t)num_alpn_protocols, &c->handshaker_factory); + config->pem_key_cert_pairs, config->num_key_cert_pairs, + config->pem_root_certs, get_tsi_client_certificate_request_type( + config->client_certificate_request), + ssl_cipher_suites(), alpn_protocol_strings, (uint16_t)num_alpn_protocols, + &c->handshaker_factory); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Handshaker factory creation failed with %s.", tsi_result_to_string(result)); @@ -895,11 +883,9 @@ grpc_security_status grpc_ssl_server_security_connector_create( c->base.add_handshakers = ssl_server_add_handshakers; *sc = &c->base; gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_OK; error: gpr_free((void *)alpn_protocol_strings); - gpr_free(alpn_protocol_string_lengths); return GRPC_SECURITY_ERROR; } diff --git a/src/core/lib/security/transport/security_connector.h b/src/core/lib/security/transport/security_connector.h index cf56cb31830..d74f6739c03 100644 --- a/src/core/lib/security/transport/security_connector.h +++ b/src/core/lib/security/transport/security_connector.h @@ -34,11 +34,14 @@ #ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H #define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H +#include + #include #include "src/core/lib/channel/handshaker.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/tcp_server.h" +#include "src/core/tsi/ssl_transport_security.h" #include "src/core/tsi/transport_security_interface.h" /* --- status enum. --- */ @@ -184,13 +187,10 @@ grpc_server_security_connector *grpc_fake_server_security_connector_create( void); /* Config for ssl clients. */ + typedef struct { - unsigned char *pem_private_key; - size_t pem_private_key_size; - unsigned char *pem_cert_chain; - size_t pem_cert_chain_size; - unsigned char *pem_root_certs; - size_t pem_root_certs_size; + tsi_ssl_pem_key_cert_pair pem_key_cert_pair; + char *pem_root_certs; } grpc_ssl_config; /* Creates an SSL channel_security_connector. @@ -211,21 +211,17 @@ grpc_security_status grpc_ssl_channel_security_connector_create( const grpc_ssl_config *config, const char *target_name, const char *overridden_target_name, grpc_channel_security_connector **sc); -/* Gets the default ssl roots. */ -size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs); +/* Gets the default ssl roots. Returns NULL if not found. */ +const char *grpc_get_default_ssl_roots(void); /* Exposed for TESTING ONLY!. */ grpc_slice grpc_get_default_ssl_roots_for_testing(void); /* Config for ssl servers. */ typedef struct { - unsigned char **pem_private_keys; - size_t *pem_private_keys_sizes; - unsigned char **pem_cert_chains; - size_t *pem_cert_chains_sizes; + tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs; size_t num_key_cert_pairs; - unsigned char *pem_root_certs; - size_t pem_root_certs_size; + char *pem_root_certs; grpc_ssl_client_certificate_request_type client_certificate_request; } grpc_ssl_server_config; diff --git a/src/core/tsi/ssl_transport_security.c b/src/core/tsi/ssl_transport_security.c index 5f4705db926..f35331e4177 100644 --- a/src/core/tsi/ssl_transport_security.c +++ b/src/core/tsi/ssl_transport_security.c @@ -479,9 +479,9 @@ static tsi_result do_ssl_write(SSL *ssl, unsigned char *unprotected_bytes, } /* Loads an in-memory PEM certificate chain into the SSL context. */ -static tsi_result ssl_ctx_use_certificate_chain( - SSL_CTX *context, const unsigned char *pem_cert_chain, - size_t pem_cert_chain_size) { +static tsi_result ssl_ctx_use_certificate_chain(SSL_CTX *context, + const char *pem_cert_chain, + size_t pem_cert_chain_size) { tsi_result result = TSI_OK; X509 *certificate = NULL; BIO *pem; @@ -522,8 +522,7 @@ static tsi_result ssl_ctx_use_certificate_chain( } /* Loads an in-memory PEM private key into the SSL context. */ -static tsi_result ssl_ctx_use_private_key(SSL_CTX *context, - const unsigned char *pem_key, +static tsi_result ssl_ctx_use_private_key(SSL_CTX *context, const char *pem_key, size_t pem_key_size) { tsi_result result = TSI_OK; EVP_PKEY *private_key = NULL; @@ -549,9 +548,11 @@ static tsi_result ssl_ctx_use_private_key(SSL_CTX *context, /* Loads in-memory PEM verification certs into the SSL context and optionally returns the verification cert names (root_names can be NULL). */ -static tsi_result ssl_ctx_load_verification_certs( - SSL_CTX *context, const unsigned char *pem_roots, size_t pem_roots_size, - STACK_OF(X509_NAME) * *root_names) { +static tsi_result ssl_ctx_load_verification_certs(SSL_CTX *context, + const char *pem_roots, + size_t pem_roots_size, + STACK_OF(X509_NAME) * + *root_names) { tsi_result result = TSI_OK; size_t num_roots = 0; X509 *root = NULL; @@ -618,24 +619,25 @@ static tsi_result ssl_ctx_load_verification_certs( /* Populates the SSL context with a private key and a cert chain, and sets the cipher list and the ephemeral ECDH key. */ static tsi_result populate_ssl_context( - SSL_CTX *context, const unsigned char *pem_private_key, - size_t pem_private_key_size, const unsigned char *pem_certificate_chain, - size_t pem_certificate_chain_size, const char *cipher_list) { + SSL_CTX *context, const tsi_ssl_pem_key_cert_pair *key_cert_pair, + const char *cipher_list) { tsi_result result = TSI_OK; - if (pem_certificate_chain != NULL) { - result = ssl_ctx_use_certificate_chain(context, pem_certificate_chain, - pem_certificate_chain_size); - if (result != TSI_OK) { - gpr_log(GPR_ERROR, "Invalid cert chain file."); - return result; + if (key_cert_pair != NULL) { + if (key_cert_pair->cert_chain != NULL) { + result = ssl_ctx_use_certificate_chain(context, key_cert_pair->cert_chain, + strlen(key_cert_pair->cert_chain)); + if (result != TSI_OK) { + gpr_log(GPR_ERROR, "Invalid cert chain file."); + return result; + } } - } - if (pem_private_key != NULL) { - result = - ssl_ctx_use_private_key(context, pem_private_key, pem_private_key_size); - if (result != TSI_OK || !SSL_CTX_check_private_key(context)) { - gpr_log(GPR_ERROR, "Invalid private key."); - return result != TSI_OK ? result : TSI_INVALID_ARGUMENT; + if (key_cert_pair->private_key != NULL) { + result = ssl_ctx_use_private_key(context, key_cert_pair->private_key, + strlen(key_cert_pair->private_key)); + if (result != TSI_OK || !SSL_CTX_check_private_key(context)) { + gpr_log(GPR_ERROR, "Invalid private key."); + return result != TSI_OK ? result : TSI_INVALID_ARGUMENT; + } } } if ((cipher_list != NULL) && !SSL_CTX_set_cipher_list(context, cipher_list)) { @@ -656,13 +658,12 @@ static tsi_result populate_ssl_context( } /* Extracts the CN and the SANs from an X509 cert as a peer object. */ -static tsi_result extract_x509_subject_names_from_pem_cert( - const unsigned char *pem_cert, size_t pem_cert_size, tsi_peer *peer) { +static tsi_result extract_x509_subject_names_from_pem_cert(const char *pem_cert, + tsi_peer *peer) { tsi_result result = TSI_OK; X509 *cert = NULL; BIO *pem; - GPR_ASSERT(pem_cert_size <= INT_MAX); - pem = BIO_new_mem_buf((void *)pem_cert, (int)pem_cert_size); + pem = BIO_new_mem_buf((void *)pem_cert, (int)strlen(pem_cert)); if (pem == NULL) return TSI_OUT_OF_RESOURCES; cert = PEM_read_bio_X509(pem, NULL, NULL, ""); @@ -679,8 +680,7 @@ static tsi_result extract_x509_subject_names_from_pem_cert( /* Builds the alpn protocol name list according to rfc 7301. */ static tsi_result build_alpn_protocol_name_list( - const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, + const char **alpn_protocols, uint16_t num_alpn_protocols, unsigned char **protocol_name_list, size_t *protocol_name_list_length) { uint16_t i; unsigned char *current; @@ -688,19 +688,21 @@ static tsi_result build_alpn_protocol_name_list( *protocol_name_list_length = 0; if (num_alpn_protocols == 0) return TSI_INVALID_ARGUMENT; for (i = 0; i < num_alpn_protocols; i++) { - if (alpn_protocols_lengths[i] == 0) { - gpr_log(GPR_ERROR, "Invalid 0-length protocol name."); + size_t length = alpn_protocols[i] == NULL ? 0 : strlen(alpn_protocols[i]); + if (length == 0 || length > 255) { + gpr_log(GPR_ERROR, "Invalid protocol name length: %d.", (int)length); return TSI_INVALID_ARGUMENT; } - *protocol_name_list_length += (size_t)alpn_protocols_lengths[i] + 1; + *protocol_name_list_length += length + 1; } *protocol_name_list = gpr_malloc(*protocol_name_list_length); if (*protocol_name_list == NULL) return TSI_OUT_OF_RESOURCES; current = *protocol_name_list; for (i = 0; i < num_alpn_protocols; i++) { - *(current++) = alpn_protocols_lengths[i]; - memcpy(current, alpn_protocols[i], alpn_protocols_lengths[i]); - current += alpn_protocols_lengths[i]; + size_t length = strlen(alpn_protocols[i]); + *(current++) = (uint8_t)length; /* max checked above. */ + memcpy(current, alpn_protocols[i], length); + current += length; } /* Safety check. */ if ((current < *protocol_name_list) || @@ -1280,11 +1282,9 @@ static int server_handshaker_factory_npn_advertised_callback( /* --- tsi_ssl_handshaker_factory constructors. --- */ tsi_result tsi_create_ssl_client_handshaker_factory( - const unsigned char *pem_private_key, size_t pem_private_key_size, - const unsigned char *pem_cert_chain, size_t pem_cert_chain_size, - const unsigned char *pem_root_certs, size_t pem_root_certs_size, - const char *cipher_list, const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, + const tsi_ssl_pem_key_cert_pair *pem_key_cert_pair, + const char *pem_root_certs, const char *cipher_suites, + const char **alpn_protocols, uint16_t num_alpn_protocols, tsi_ssl_client_handshaker_factory **factory) { SSL_CTX *ssl_context = NULL; tsi_ssl_client_handshaker_factory *impl = NULL; @@ -1307,20 +1307,19 @@ tsi_result tsi_create_ssl_client_handshaker_factory( do { result = - populate_ssl_context(ssl_context, pem_private_key, pem_private_key_size, - pem_cert_chain, pem_cert_chain_size, cipher_list); + populate_ssl_context(ssl_context, pem_key_cert_pair, cipher_suites); if (result != TSI_OK) break; result = ssl_ctx_load_verification_certs(ssl_context, pem_root_certs, - pem_root_certs_size, NULL); + strlen(pem_root_certs), NULL); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Cannot load server root certificates."); break; } if (num_alpn_protocols != 0) { - result = build_alpn_protocol_name_list( - alpn_protocols, alpn_protocols_lengths, num_alpn_protocols, - &impl->alpn_protocol_list, &impl->alpn_protocol_list_length); + result = build_alpn_protocol_name_list(alpn_protocols, num_alpn_protocols, + &impl->alpn_protocol_list, + &impl->alpn_protocol_list_length); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Building alpn list failed with error %s.", tsi_result_to_string(result)); @@ -1352,34 +1351,24 @@ tsi_result tsi_create_ssl_client_handshaker_factory( } tsi_result tsi_create_ssl_server_handshaker_factory( - const unsigned char **pem_private_keys, - const size_t *pem_private_keys_sizes, const unsigned char **pem_cert_chains, - const size_t *pem_cert_chains_sizes, size_t key_cert_pair_count, - const unsigned char *pem_client_root_certs, - size_t pem_client_root_certs_size, int force_client_auth, - const char *cipher_list, const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, + const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, + size_t num_key_cert_pairs, const char *pem_client_root_certs, + int force_client_auth, const char *cipher_suites, + const char **alpn_protocols, uint16_t num_alpn_protocols, tsi_ssl_server_handshaker_factory **factory) { return tsi_create_ssl_server_handshaker_factory_ex( - pem_private_keys, pem_private_keys_sizes, pem_cert_chains, - pem_cert_chains_sizes, key_cert_pair_count, pem_client_root_certs, - pem_client_root_certs_size, + pem_key_cert_pairs, num_key_cert_pairs, pem_client_root_certs, force_client_auth ? TSI_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY : TSI_DONT_REQUEST_CLIENT_CERTIFICATE, - cipher_list, alpn_protocols, alpn_protocols_lengths, num_alpn_protocols, - factory); + cipher_suites, alpn_protocols, num_alpn_protocols, factory); } tsi_result tsi_create_ssl_server_handshaker_factory_ex( - const unsigned char **pem_private_keys, - const size_t *pem_private_keys_sizes, const unsigned char **pem_cert_chains, - const size_t *pem_cert_chains_sizes, size_t key_cert_pair_count, - const unsigned char *pem_client_root_certs, - size_t pem_client_root_certs_size, + const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, + size_t num_key_cert_pairs, const char *pem_client_root_certs, tsi_client_certificate_request_type client_certificate_request, - const char *cipher_list, const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, - tsi_ssl_server_handshaker_factory **factory) { + const char *cipher_suites, const char **alpn_protocols, + uint16_t num_alpn_protocols, tsi_ssl_server_handshaker_factory **factory) { tsi_ssl_server_handshaker_factory *impl = NULL; tsi_result result = TSI_OK; size_t i = 0; @@ -1388,33 +1377,32 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( if (factory == NULL) return TSI_INVALID_ARGUMENT; *factory = NULL; - if (key_cert_pair_count == 0 || pem_private_keys == NULL || - pem_cert_chains == NULL) { + if (num_key_cert_pairs == 0 || pem_key_cert_pairs == NULL) { return TSI_INVALID_ARGUMENT; } impl = gpr_zalloc(sizeof(*impl)); - impl->ssl_contexts = gpr_zalloc(key_cert_pair_count * sizeof(SSL_CTX *)); + impl->ssl_contexts = gpr_zalloc(num_key_cert_pairs * sizeof(SSL_CTX *)); impl->ssl_context_x509_subject_names = - gpr_zalloc(key_cert_pair_count * sizeof(tsi_peer)); + gpr_zalloc(num_key_cert_pairs * sizeof(tsi_peer)); if (impl->ssl_contexts == NULL || impl->ssl_context_x509_subject_names == NULL) { tsi_ssl_server_handshaker_factory_destroy(impl); return TSI_OUT_OF_RESOURCES; } - impl->ssl_context_count = key_cert_pair_count; + impl->ssl_context_count = num_key_cert_pairs; if (num_alpn_protocols > 0) { - result = build_alpn_protocol_name_list( - alpn_protocols, alpn_protocols_lengths, num_alpn_protocols, - &impl->alpn_protocol_list, &impl->alpn_protocol_list_length); + result = build_alpn_protocol_name_list(alpn_protocols, num_alpn_protocols, + &impl->alpn_protocol_list, + &impl->alpn_protocol_list_length); if (result != TSI_OK) { tsi_ssl_server_handshaker_factory_destroy(impl); return result; } } - for (i = 0; i < key_cert_pair_count; i++) { + for (i = 0; i < num_key_cert_pairs; i++) { do { impl->ssl_contexts[i] = SSL_CTX_new(TLSv1_2_method()); if (impl->ssl_contexts[i] == NULL) { @@ -1422,16 +1410,15 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( result = TSI_OUT_OF_RESOURCES; break; } - result = populate_ssl_context( - impl->ssl_contexts[i], pem_private_keys[i], pem_private_keys_sizes[i], - pem_cert_chains[i], pem_cert_chains_sizes[i], cipher_list); + result = populate_ssl_context(impl->ssl_contexts[i], + &pem_key_cert_pairs[i], cipher_suites); if (result != TSI_OK) break; if (pem_client_root_certs != NULL) { STACK_OF(X509_NAME) *root_names = NULL; result = ssl_ctx_load_verification_certs( impl->ssl_contexts[i], pem_client_root_certs, - pem_client_root_certs_size, &root_names); + strlen(pem_client_root_certs), &root_names); if (result != TSI_OK) { gpr_log(GPR_ERROR, "Invalid verification certs."); break; @@ -1464,7 +1451,7 @@ tsi_result tsi_create_ssl_server_handshaker_factory_ex( } result = extract_x509_subject_names_from_pem_cert( - pem_cert_chains[i], pem_cert_chains_sizes[i], + pem_key_cert_pairs[i].cert_chain, &impl->ssl_context_x509_subject_names[i]); if (result != TSI_OK) break; diff --git a/src/core/tsi/ssl_transport_security.h b/src/core/tsi/ssl_transport_security.h index 48dcaec1212..3117571d9f7 100644 --- a/src/core/tsi/ssl_transport_security.h +++ b/src/core/tsi/ssl_transport_security.h @@ -60,27 +60,32 @@ extern "C" { typedef struct tsi_ssl_client_handshaker_factory tsi_ssl_client_handshaker_factory; +/* Object that holds a private key / certificate chain pair in PEM format. */ +typedef struct { + /* private_key is the NULL-terminated string containing the PEM encoding of + the client's private key. */ + const char *private_key; + + /* cert_chain is the NULL-terminated string containing the PEM encoding of + the client's certificate chain. */ + const char *cert_chain; +} tsi_ssl_pem_key_cert_pair; + /* Creates a client handshaker factory. - - pem_private_key is the buffer containing the PEM encoding of the client's - private key. This parameter can be NULL if the client does not have a - private key. - - pem_private_key_size is the size of the associated buffer. - - pem_cert_chain is the buffer containing the PEM encoding of the client's - certificate chain. This parameter can be NULL if the client does not have - a certificate chain. - - pem_cert_chain_size is the size of the associated buffer. - - pem_roots_cert is the buffer containing the PEM encoding of the server - root certificates. This parameter cannot be NULL. - - pem_roots_cert_size is the size of the associated buffer. + - pem_key_cert_pair is a pointer to the object containing client's private + key and certificate chain. This parameter can be NULL if the client does + not have such a key/cert pair. + - pem_roots_cert is the NULL-terminated string containing the PEM encoding of + the client root certificates. This parameter may be NULL if the server does + not want the client to be authenticated with SSL. - cipher_suites contains an optional list of the ciphers that the client supports. The format of this string is described in: https://www.openssl.org/docs/apps/ciphers.html. This parameter can be set to NULL to use the default set of ciphers. TODO(jboeuf): Revisit the format of this parameter. - - alpn_protocols is an array containing the protocol names that the - handshakers created with this factory support. This parameter can be NULL. - - alpn_protocols_lengths is an array containing the lengths of the alpn - protocols specified in alpn_protocols. This parameter can be NULL. + - alpn_protocols is an array containing the NULL terminated protocol names + that the handshakers created with this factory support. This parameter can + be NULL. - num_alpn_protocols is the number of alpn protocols and associated lengths specified. If this parameter is 0, the other alpn parameters must be NULL. - factory is the address of the factory pointer to be created. @@ -88,11 +93,9 @@ typedef struct tsi_ssl_client_handshaker_factory - This method returns TSI_OK on success or TSI_INVALID_PARAMETER in the case where a parameter is invalid. */ tsi_result tsi_create_ssl_client_handshaker_factory( - const unsigned char *pem_private_key, size_t pem_private_key_size, - const unsigned char *pem_cert_chain, size_t pem_cert_chain_size, - const unsigned char *pem_root_certs, size_t pem_root_certs_size, - const char *cipher_suites, const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, + const tsi_ssl_pem_key_cert_pair *pem_key_cert_pair, + const char *pem_root_certs, const char *cipher_suites, + const char **alpn_protocols, uint16_t num_alpn_protocols, tsi_ssl_client_handshaker_factory **factory); /* Creates a client handshaker. @@ -122,37 +125,19 @@ typedef struct tsi_ssl_server_handshaker_factory tsi_ssl_server_handshaker_factory; /* Creates a server handshaker factory. - - version indicates which version of the specification to use. - - pem_private_keys is an array containing the PEM encoding of the server's - private keys. This parameter cannot be NULL. The size of the array is - given by the key_cert_pair_count parameter. - - pem_private_keys_sizes is the array containing the sizes of the associated - buffers. - - pem_cert_chains is an array containing the PEM encoding of the server's - cert chains. This parameter cannot be NULL. The size of the array is - given by the key_cert_pair_count parameter. - - pem_cert_chains_sizes is the array containing the sizes of the associated - buffers. - - key_cert_pair_count indicates the number of items in the private_key_files - and cert_chain_files parameters. - - pem_client_roots is the buffer containing the PEM encoding of the client - root certificates. This parameter may be NULL in which case the server will - not authenticate the client. If not NULL, the force_client_auth parameter - specifies if the server will accept only authenticated clients or both - authenticated and non-authenticated clients. - - pem_client_root_certs_size is the size of the associated buffer. - - force_client_auth, if set to non-zero will force the client to authenticate - with an SSL cert. Note that this option is ignored if pem_client_root_certs - is NULL or pem_client_roots_certs_size is 0 + - pem_key_cert_pairs is an array private key / certificate chains of the + server. + - num_key_cert_pairs is the number of items in the pem_key_cert_pairs array. + - pem_root_certs is the NULL-terminated string containing the PEM encoding + of the server root certificates. - cipher_suites contains an optional list of the ciphers that the server supports. The format of this string is described in: https://www.openssl.org/docs/apps/ciphers.html. This parameter can be set to NULL to use the default set of ciphers. TODO(jboeuf): Revisit the format of this parameter. - - alpn_protocols is an array containing the protocol names that the - handshakers created with this factory support. This parameter can be NULL. - - alpn_protocols_lengths is an array containing the lengths of the alpn - protocols specified in alpn_protocols. This parameter can be NULL. + - alpn_protocols is an array containing the NULL terminated protocol names + that the handshakers created with this factory support. This parameter can + be NULL. - num_alpn_protocols is the number of alpn protocols and associated lengths specified. If this parameter is 0, the other alpn parameters must be NULL. - factory is the address of the factory pointer to be created. @@ -160,13 +145,10 @@ typedef struct tsi_ssl_server_handshaker_factory - This method returns TSI_OK on success or TSI_INVALID_PARAMETER in the case where a parameter is invalid. */ tsi_result tsi_create_ssl_server_handshaker_factory( - const unsigned char **pem_private_keys, - const size_t *pem_private_keys_sizes, const unsigned char **pem_cert_chains, - const size_t *pem_cert_chains_sizes, size_t key_cert_pair_count, - const unsigned char *pem_client_root_certs, - size_t pem_client_root_certs_size, int force_client_auth, - const char *cipher_suites, const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, + const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, + size_t num_key_cert_pairs, const char *pem_client_root_certs, + int force_client_auth, const char *cipher_suites, + const char **alpn_protocols, uint16_t num_alpn_protocols, tsi_ssl_server_handshaker_factory **factory); /* Same as tsi_create_ssl_server_handshaker_factory method except uses @@ -176,15 +158,11 @@ tsi_result tsi_create_ssl_server_handshaker_factory( authenticate with an SSL cert. Note that this option is ignored if pem_client_root_certs is NULL or pem_client_roots_certs_size is 0 */ tsi_result tsi_create_ssl_server_handshaker_factory_ex( - const unsigned char **pem_private_keys, - const size_t *pem_private_keys_sizes, const unsigned char **pem_cert_chains, - const size_t *pem_cert_chains_sizes, size_t key_cert_pair_count, - const unsigned char *pem_client_root_certs, - size_t pem_client_root_certs_size, + const tsi_ssl_pem_key_cert_pair *pem_key_cert_pairs, + size_t num_key_cert_pairs, const char *pem_client_root_certs, tsi_client_certificate_request_type client_certificate_request, - const char *cipher_suites, const unsigned char **alpn_protocols, - const unsigned char *alpn_protocols_lengths, uint16_t num_alpn_protocols, - tsi_ssl_server_handshaker_factory **factory); + const char *cipher_suites, const char **alpn_protocols, + uint16_t num_alpn_protocols, tsi_ssl_server_handshaker_factory **factory); /* Creates a server handshaker. - self is the factory from which the handshaker will be created. From 78ca0d1d57248e07eebb028e9fe2e587076d5fd1 Mon Sep 17 00:00:00 2001 From: Mahak Mukhi Date: Wed, 26 Apr 2017 13:39:46 -0700 Subject: [PATCH 54/60] Fix for recent cmake failure owing to gmock --- CMakeLists.txt | 243 +++++++++++++++++++++++++++++- templates/CMakeLists.txt.template | 13 +- 2 files changed, 252 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d7662031d1..1306884d706 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -316,10 +316,11 @@ function(protobuf_generate_grpc_cpp) add_custom_command( OUTPUT "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" + "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" COMMAND ${_gRPC_PROTOBUF_PROTOC} - ARGS --grpc_out=${_gRPC_PROTO_GENS_DIR} + ARGS --grpc_out=generate_mock_code=true:${_gRPC_PROTO_GENS_DIR} --cpp_out=${_gRPC_PROTO_GENS_DIR} --plugin=protoc-gen-grpc=$ ${_protobuf_include_path} @@ -329,7 +330,7 @@ function(protobuf_generate_grpc_cpp) COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}" VERBATIM) - set_source_files_properties("${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" PROPERTIES GENERATED TRUE) + set_source_files_properties("${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" PROPERTIES GENERATED TRUE) endforeach() endfunction() @@ -3008,6 +3009,8 @@ target_include_directories(grpc++_proto_reflection_desc_db PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3127,6 +3130,8 @@ target_include_directories(grpc++_test_config PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3152,6 +3157,7 @@ add_library(grpc++_test_util ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.pb.h ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.h + ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_mock.grpc.pb.h ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.cc ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/duplicate/echo_duplicate.pb.h @@ -3204,6 +3210,8 @@ target_include_directories(grpc++_test_util PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3630,6 +3638,8 @@ target_include_directories(grpc_benchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3689,6 +3699,8 @@ target_include_directories(grpc_cli_libs PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3828,6 +3840,8 @@ target_include_directories(http2_client_main PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3883,6 +3897,8 @@ target_include_directories(interop_client_helper PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -3953,6 +3969,8 @@ target_include_directories(interop_client_main PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -4004,6 +4022,8 @@ target_include_directories(interop_server_helper PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -4073,6 +4093,8 @@ target_include_directories(interop_server_lib PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -4124,6 +4146,8 @@ target_include_directories(interop_server_main PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -4212,6 +4236,8 @@ target_include_directories(qps PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -8767,6 +8793,7 @@ if (gRPC_BUILD_TESTS) add_executable(alarm_cpp_test test/cpp/common/alarm_cpp_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -8785,6 +8812,8 @@ target_include_directories(alarm_cpp_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -8806,6 +8835,7 @@ if (gRPC_BUILD_TESTS) add_executable(async_end2end_test test/cpp/end2end/async_end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -8824,6 +8854,8 @@ target_include_directories(async_end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -8845,6 +8877,7 @@ if (gRPC_BUILD_TESTS) add_executable(auth_property_iterator_test test/cpp/common/auth_property_iterator_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -8863,6 +8896,8 @@ target_include_directories(auth_property_iterator_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -8885,6 +8920,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_arena test/cpp/microbenchmarks/bm_arena.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -8903,6 +8939,8 @@ target_include_directories(bm_arena PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -8928,6 +8966,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_call_create test/cpp/microbenchmarks/bm_call_create.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -8946,6 +8985,8 @@ target_include_directories(bm_call_create PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -8971,6 +9012,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_chttp2_hpack test/cpp/microbenchmarks/bm_chttp2_hpack.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -8989,6 +9031,8 @@ target_include_directories(bm_chttp2_hpack PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9014,6 +9058,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_chttp2_transport test/cpp/microbenchmarks/bm_chttp2_transport.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9032,6 +9077,8 @@ target_include_directories(bm_chttp2_transport PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9057,6 +9104,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_closure test/cpp/microbenchmarks/bm_closure.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9075,6 +9123,8 @@ target_include_directories(bm_closure PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9100,6 +9150,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_cq test/cpp/microbenchmarks/bm_cq.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9118,6 +9169,8 @@ target_include_directories(bm_cq PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9143,6 +9196,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_cq_multiple_threads test/cpp/microbenchmarks/bm_cq_multiple_threads.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9161,6 +9215,8 @@ target_include_directories(bm_cq_multiple_threads PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9186,6 +9242,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_error test/cpp/microbenchmarks/bm_error.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9204,6 +9261,8 @@ target_include_directories(bm_error PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9229,6 +9288,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_fullstack_streaming_ping_pong test/cpp/microbenchmarks/bm_fullstack_streaming_ping_pong.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9247,6 +9307,8 @@ target_include_directories(bm_fullstack_streaming_ping_pong PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9272,6 +9334,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_fullstack_streaming_pump test/cpp/microbenchmarks/bm_fullstack_streaming_pump.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9290,6 +9353,8 @@ target_include_directories(bm_fullstack_streaming_pump PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9315,6 +9380,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_fullstack_trickle test/cpp/microbenchmarks/bm_fullstack_trickle.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9333,6 +9399,8 @@ target_include_directories(bm_fullstack_trickle PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9358,6 +9426,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_fullstack_unary_ping_pong test/cpp/microbenchmarks/bm_fullstack_unary_ping_pong.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9376,6 +9445,8 @@ target_include_directories(bm_fullstack_unary_ping_pong PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9401,6 +9472,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_metadata test/cpp/microbenchmarks/bm_metadata.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9419,6 +9491,8 @@ target_include_directories(bm_metadata PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9444,6 +9518,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(bm_pollset test/cpp/microbenchmarks/bm_pollset.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9462,6 +9537,8 @@ target_include_directories(bm_pollset PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9486,6 +9563,7 @@ if (gRPC_BUILD_TESTS) add_executable(channel_arguments_test test/cpp/common/channel_arguments_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9504,6 +9582,8 @@ target_include_directories(channel_arguments_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9522,6 +9602,7 @@ if (gRPC_BUILD_TESTS) add_executable(channel_filter_test test/cpp/common/channel_filter_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9540,6 +9621,8 @@ target_include_directories(channel_filter_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9558,6 +9641,7 @@ if (gRPC_BUILD_TESTS) add_executable(cli_call_test test/cpp/util/cli_call_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9576,6 +9660,8 @@ target_include_directories(cli_call_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9599,6 +9685,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(client_crash_test test/cpp/end2end/client_crash_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9617,6 +9704,8 @@ target_include_directories(client_crash_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9639,6 +9728,7 @@ if (gRPC_BUILD_TESTS) add_executable(client_crash_test_server test/cpp/end2end/client_crash_test_server.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9657,6 +9747,8 @@ target_include_directories(client_crash_test_server PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9698,6 +9790,7 @@ add_executable(codegen_test_full ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/stats.grpc.pb.h test/cpp/codegen/codegen_test_full.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -9731,6 +9824,8 @@ target_include_directories(codegen_test_full PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9770,6 +9865,7 @@ add_executable(codegen_test_minimal test/cpp/codegen/codegen_test_minimal.cc src/cpp/codegen/codegen_init.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -9803,6 +9899,8 @@ target_include_directories(codegen_test_minimal PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9820,6 +9918,7 @@ if (gRPC_BUILD_TESTS) add_executable(credentials_test test/cpp/client/credentials_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9838,6 +9937,8 @@ target_include_directories(credentials_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9856,6 +9957,7 @@ if (gRPC_BUILD_TESTS) add_executable(cxx_byte_buffer_test test/cpp/util/byte_buffer_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9874,6 +9976,8 @@ target_include_directories(cxx_byte_buffer_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9894,6 +9998,7 @@ if (gRPC_BUILD_TESTS) add_executable(cxx_slice_test test/cpp/util/slice_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9912,6 +10017,8 @@ target_include_directories(cxx_slice_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9932,6 +10039,7 @@ if (gRPC_BUILD_TESTS) add_executable(cxx_string_ref_test test/cpp/util/string_ref_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9950,6 +10058,8 @@ target_include_directories(cxx_string_ref_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -9966,6 +10076,7 @@ if (gRPC_BUILD_TESTS) add_executable(cxx_time_test test/cpp/util/time_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -9984,6 +10095,8 @@ target_include_directories(cxx_time_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10004,6 +10117,7 @@ if (gRPC_BUILD_TESTS) add_executable(end2end_test test/cpp/end2end/end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10022,6 +10136,8 @@ target_include_directories(end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10047,6 +10163,7 @@ add_executable(error_details_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.h test/cpp/util/error_details_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -10068,6 +10185,8 @@ target_include_directories(error_details_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10085,6 +10204,7 @@ if (gRPC_BUILD_TESTS) add_executable(filter_end2end_test test/cpp/end2end/filter_end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10103,6 +10223,8 @@ target_include_directories(filter_end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10124,6 +10246,7 @@ if (gRPC_BUILD_TESTS) add_executable(generic_end2end_test test/cpp/end2end/generic_end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10142,6 +10265,8 @@ target_include_directories(generic_end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10167,6 +10292,7 @@ add_executable(golden_file_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/compiler_test.grpc.pb.h test/cpp/codegen/golden_file_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -10188,6 +10314,8 @@ target_include_directories(golden_file_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10206,6 +10334,7 @@ if (gRPC_BUILD_TESTS) add_executable(grpc_cli test/cpp/util/grpc_cli.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10224,6 +10353,8 @@ target_include_directories(grpc_cli PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10519,6 +10650,7 @@ add_executable(grpc_tool_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo_messages.grpc.pb.h test/cpp/util/grpc_tool_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -10543,6 +10675,8 @@ target_include_directories(grpc_tool_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10571,6 +10705,7 @@ add_executable(grpclb_api_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/lb/v1/load_balancer.grpc.pb.h test/cpp/grpclb/grpclb_api_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -10592,6 +10727,8 @@ target_include_directories(grpclb_api_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10615,6 +10752,7 @@ add_executable(grpclb_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/lb/v1/load_balancer.grpc.pb.h test/cpp/grpclb/grpclb_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -10636,6 +10774,8 @@ target_include_directories(grpclb_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10657,6 +10797,7 @@ if (gRPC_BUILD_TESTS) add_executable(health_service_end2end_test test/cpp/end2end/health_service_end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10675,6 +10816,8 @@ target_include_directories(health_service_end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10696,6 +10839,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(http2_client third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10714,6 +10858,8 @@ target_include_directories(http2_client PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10736,6 +10882,7 @@ if (gRPC_BUILD_TESTS) add_executable(hybrid_end2end_test test/cpp/end2end/hybrid_end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10754,6 +10901,8 @@ target_include_directories(hybrid_end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10775,6 +10924,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(interop_client third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10793,6 +10943,8 @@ target_include_directories(interop_client PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10818,6 +10970,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(interop_server third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10836,6 +10989,8 @@ target_include_directories(interop_server PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10863,6 +11018,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(interop_test test/cpp/interop/interop_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10881,6 +11037,8 @@ target_include_directories(interop_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10903,6 +11061,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(json_run_localhost test/cpp/qps/json_run_localhost.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -10921,6 +11080,8 @@ target_include_directories(json_run_localhost PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10948,6 +11109,7 @@ add_executable(metrics_client ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/metrics.grpc.pb.h test/cpp/interop/metrics_client.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -10969,6 +11131,8 @@ target_include_directories(metrics_client PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -10988,6 +11152,7 @@ if (gRPC_BUILD_TESTS) add_executable(mock_test test/cpp/end2end/mock_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11006,6 +11171,8 @@ target_include_directories(mock_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11027,6 +11194,7 @@ if (gRPC_BUILD_TESTS) add_executable(noop-benchmark test/cpp/microbenchmarks/noop-benchmark.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11045,6 +11213,8 @@ target_include_directories(noop-benchmark PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11061,6 +11231,7 @@ if (gRPC_BUILD_TESTS) add_executable(proto_server_reflection_test test/cpp/end2end/proto_server_reflection_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11079,6 +11250,8 @@ target_include_directories(proto_server_reflection_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11102,6 +11275,7 @@ if (gRPC_BUILD_TESTS) add_executable(proto_utils_test test/cpp/codegen/proto_utils_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11120,6 +11294,8 @@ target_include_directories(proto_utils_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11138,6 +11314,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(qps_interarrival_test test/cpp/qps/qps_interarrival_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11156,6 +11333,8 @@ target_include_directories(qps_interarrival_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11179,6 +11358,7 @@ if (gRPC_BUILD_TESTS) add_executable(qps_json_driver test/cpp/qps/qps_json_driver.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11197,6 +11377,8 @@ target_include_directories(qps_json_driver PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11221,6 +11403,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(qps_openloop_test test/cpp/qps/qps_openloop_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11239,6 +11422,8 @@ target_include_directories(qps_openloop_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11263,6 +11448,7 @@ if (gRPC_BUILD_TESTS) add_executable(qps_worker test/cpp/qps/worker.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11281,6 +11467,8 @@ target_include_directories(qps_worker PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11316,6 +11504,7 @@ add_executable(reconnect_interop_client ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/test.grpc.pb.h test/cpp/interop/reconnect_interop_client.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -11343,6 +11532,8 @@ target_include_directories(reconnect_interop_client PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11377,6 +11568,7 @@ add_executable(reconnect_interop_server ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/test.grpc.pb.h test/cpp/interop/reconnect_interop_server.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -11404,6 +11596,8 @@ target_include_directories(reconnect_interop_server PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11428,6 +11622,7 @@ if (gRPC_BUILD_TESTS) add_executable(round_robin_end2end_test test/cpp/end2end/round_robin_end2end_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11446,6 +11641,8 @@ target_include_directories(round_robin_end2end_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11467,6 +11664,7 @@ if (gRPC_BUILD_TESTS) add_executable(secure_auth_context_test test/cpp/common/secure_auth_context_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11485,6 +11683,8 @@ target_include_directories(secure_auth_context_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11507,6 +11707,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(secure_sync_unary_ping_pong_test test/cpp/qps/secure_sync_unary_ping_pong_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11525,6 +11726,8 @@ target_include_directories(secure_sync_unary_ping_pong_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11548,6 +11751,7 @@ if (gRPC_BUILD_TESTS) add_executable(server_builder_plugin_test test/cpp/end2end/server_builder_plugin_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11566,6 +11770,8 @@ target_include_directories(server_builder_plugin_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11595,6 +11801,7 @@ add_executable(server_builder_test ${_gRPC_PROTO_GENS_DIR}/src/proto/grpc/testing/echo.grpc.pb.h test/cpp/server/server_builder_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -11619,6 +11826,8 @@ target_include_directories(server_builder_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11640,6 +11849,7 @@ if (gRPC_BUILD_TESTS) add_executable(server_context_test_spouse_test test/cpp/test/server_context_test_spouse_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11658,6 +11868,8 @@ target_include_directories(server_context_test_spouse_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11679,6 +11891,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(server_crash_test test/cpp/end2end/server_crash_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11697,6 +11910,8 @@ target_include_directories(server_crash_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11719,6 +11934,7 @@ if (gRPC_BUILD_TESTS) add_executable(server_crash_test_client test/cpp/end2end/server_crash_test_client.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11737,6 +11953,8 @@ target_include_directories(server_crash_test_client PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11758,6 +11976,7 @@ if (gRPC_BUILD_TESTS) add_executable(shutdown_test test/cpp/end2end/shutdown_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11776,6 +11995,8 @@ target_include_directories(shutdown_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11797,6 +12018,7 @@ if (gRPC_BUILD_TESTS) add_executable(status_test test/cpp/util/status_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11815,6 +12037,8 @@ target_include_directories(status_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11836,6 +12060,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(streaming_throughput_test test/cpp/end2end/streaming_throughput_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11854,6 +12079,8 @@ target_include_directories(streaming_throughput_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11895,6 +12122,7 @@ add_executable(stress_test test/cpp/interop/stress_test.cc test/cpp/util/metrics_server.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) protobuf_generate_grpc_cpp( @@ -11925,6 +12153,8 @@ target_include_directories(stress_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11947,6 +12177,7 @@ if (gRPC_BUILD_TESTS) add_executable(thread_manager_test test/cpp/thread_manager/thread_manager_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -11965,6 +12196,8 @@ target_include_directories(thread_manager_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -11984,6 +12217,7 @@ if (gRPC_BUILD_TESTS) add_executable(thread_stress_test test/cpp/end2end/thread_stress_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -12002,6 +12236,8 @@ target_include_directories(thread_stress_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) @@ -12024,6 +12260,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(writes_per_rpc_test test/cpp/performance/writes_per_rpc_test.cc third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc ) @@ -12042,6 +12279,8 @@ target_include_directories(writes_per_rpc_test PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock PRIVATE ${_gRPC_PROTO_GENS_DIR} ) diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index 88e518f1321..2252a7ea80a 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -361,10 +361,11 @@ add_custom_command( OUTPUT <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" + <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" <%text>"${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" COMMAND <%text>${_gRPC_PROTOBUF_PROTOC} - ARGS --grpc_out=<%text>${_gRPC_PROTO_GENS_DIR} + ARGS --grpc_out=<%text>generate_mock_code=true:${_gRPC_PROTO_GENS_DIR} --cpp_out=<%text>${_gRPC_PROTO_GENS_DIR} --plugin=protoc-gen-grpc=$ <%text>${_protobuf_include_path} @@ -374,7 +375,7 @@ COMMENT "Running gRPC C++ protocol buffer compiler on <%text>${FIL}" VERBATIM) - <%text>set_source_files_properties("${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" PROPERTIES GENERATED TRUE) + <%text>set_source_files_properties("${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}_mock.grpc.pb.h" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.cc" "${_gRPC_PROTO_GENS_DIR}/${RELFIL_WE}.pb.h" PROPERTIES GENERATED TRUE) endforeach() endfunction() @@ -475,6 +476,9 @@ ${proto_replace_ext(src, '.grpc.pb.cc')} ${proto_replace_ext(src, '.pb.h')} ${proto_replace_ext(src, '.grpc.pb.h')} + % if src in ["src/proto/grpc/testing/compiler_test.proto", "src/proto/grpc/testing/echo.proto"]: + ${proto_replace_ext(src, '_mock.grpc.pb.h')} + % endif % endif % endfor ) @@ -514,6 +518,8 @@ % if lib.build in ['test', 'private'] and lib.language == 'c++': PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock % endif % if lib.language == 'c++': PRIVATE <%text>${_gRPC_PROTO_GENS_DIR} @@ -557,6 +563,7 @@ % endfor % if tgt.build == 'test' and tgt.language == 'c++': third_party/googletest/googletest/src/gtest-all.cc + third_party/googletest/googlemock/src/gmock-all.cc % endif ) @@ -584,6 +591,8 @@ % if tgt.build in ['test', 'private'] and tgt.language == 'c++': PRIVATE third_party/googletest/googletest/include PRIVATE third_party/googletest/googletest + PRIVATE third_party/googletest/googlemock/include + PRIVATE third_party/googletest/googlemock % endif % if tgt.language == 'c++': PRIVATE <%text>${_gRPC_PROTO_GENS_DIR} From 6ae9175690f26b21b2888c780821482074d4a1bb Mon Sep 17 00:00:00 2001 From: Matt Kwong Date: Wed, 26 Apr 2017 19:03:18 -0700 Subject: [PATCH 55/60] Update performance VM init script --- tools/gce/create_linux_performance_worker.sh | 4 +- tools/gce/linux_performance_worker_init.sh | 50 +++++++++++--------- 2 files changed, 29 insertions(+), 25 deletions(-) diff --git a/tools/gce/create_linux_performance_worker.sh b/tools/gce/create_linux_performance_worker.sh index 2c8cf0b96b3..68710e13b07 100755 --- a/tools/gce/create_linux_performance_worker.sh +++ b/tools/gce/create_linux_performance_worker.sh @@ -42,14 +42,14 @@ CLOUD_PROJECT=grpc-testing ZONE=us-central1-b # this zone allows 32core machines INSTANCE_NAME="${1:-grpc-performance-server1}" -MACHINE_TYPE=n1-standard-8 +MACHINE_TYPE=n1-standard-32 gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ --machine-type $MACHINE_TYPE \ --image-project ubuntu-os-cloud \ - --image-family ubuntu-1604-lts \ + --image-family ubuntu-1610 \ --boot-disk-size 300 \ --scopes https://www.googleapis.com/auth/bigquery diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 17f36fb4ff8..78cdd31f0b0 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -55,7 +55,10 @@ sudo apt-get install -y \ libc6 \ libc6-dbg \ libc6-dev \ + libcurl4-openssl-dev \ libgtest-dev \ + libreadline-dev \ + libssl-dev \ libtool \ make \ strace \ @@ -71,7 +74,8 @@ sudo apt-get install -y \ telnet \ unzip \ wget \ - zip + zip \ + zlib1g-dev # perftools sudo apt-get install -y google-perftools libgoogle-perftools-dev @@ -87,14 +91,15 @@ sudo pip install tabulate sudo pip install google-api-python-client sudo pip install virtualenv -# TODO(jtattermusch): For some reason, building gRPC Python depends on python3.4 -# being installed, but python3.4 is not available on Ubuntu 16.04. -# Temporarily fixing this by adding a PPA with python3.4, but we should -# really remove this hack once possible. -sudo add-apt-repository -y ppa:fkrull/deadsnakes -sudo apt-get update -sudo apt-get install -y python3.4 python3.4-dev -python3.4 -m pip install virtualenv +# Building gRPC Python depends on python3.4 being installed, but python3.4 +# is not available on Ubuntu 16.10, so install from source +curl -O https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz +tar xzvf Python-3.4.6.tgz +cd Python-3.4.6 +./configure --enable-shared --prefix=/usr/local LDFLAGS="-Wl,--rpath=/usr/local/lib" +sudo make altinstall +cd .. +rm Python-3.4.6.tgz curl -O https://bootstrap.pypa.io/get-pip.py sudo pypy get-pip.py @@ -117,18 +122,25 @@ sudo apt-get update sudo apt-get install -y mono-devel nuget # C# .NET Core dependencies (https://www.microsoft.com/net/core#ubuntu) -sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ xenial main" > /etc/apt/sources.list.d/dotnetdev.list' +sudo sh -c 'echo "deb [arch=amd64] https://apt-mo.trafficmanager.net/repos/dotnet-release/ yakkety main" > /etc/apt/sources.list.d/dotnetdev.list' sudo apt-key adv --keyserver apt-mo.trafficmanager.net --recv-keys 417A0893 sudo apt-get update -sudo apt-get install -y dotnet-dev-1.0.0-preview2-003131 +sudo apt-get install -y dotnet-dev-1.0.0-preview2.1-003155 sudo apt-get install -y dotnet-dev-1.0.1 # Ruby dependencies -gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 -curl -sSL https://get.rvm.io | bash -s stable --ruby +git clone https://github.com/rbenv/rbenv.git ~/.rbenv +export PATH="$HOME/.rbenv/bin:$PATH" +eval "$(rbenv init -)" + +git clone https://github.com/rbenv/ruby-build.git ~/.rbenv/plugins/ruby-build +export PATH="$HOME/.rbenv/plugins/ruby-build/bin:$PATH" + +rbenv install 2.4.0 +rbenv global 2.4.0 +ruby -v # Install bundler (prerequisite for gRPC Ruby) -source ~/.rvm/scripts/rvm gem install bundler # Java dependencies - nothing as we already have Java JDK 8 @@ -163,15 +175,7 @@ echo 4096 | sudo tee /proc/sys/kernel/perf_event_mlock_kb git clone -v https://github.com/brendangregg/FlameGraph ~/FlameGraph # Install scipy and numpy for benchmarking scripts -sudo apt-get install python-scipy python-numpy - -# Update Linux kernel to 4.9 -wget \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920_4.9.20-040920.201703310531_all.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-headers-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb \ - kernel.ubuntu.com/~kernel-ppa/mainline/v4.9.20/linux-image-4.9.20-040920-generic_4.9.20-040920.201703310531_amd64.deb -sudo dpkg -i linux-headers-4.9*.deb linux-image-4.9*.deb -rm linux-* +sudo apt-get install -y python-scipy python-numpy # Add pubkey of jenkins@grpc-jenkins-master to authorized keys of jenkins@ # This needs to happen as the last step to prevent Jenkins master from connecting From 815e3182680c37b2bb4abb57dc4908d2b4169a0f Mon Sep 17 00:00:00 2001 From: Julien Boeuf Date: Thu, 20 Apr 2017 13:20:32 -0700 Subject: [PATCH 56/60] Removing cipher suites blacklisted in HTTP2. See http://httpwg.org/specs/rfc7540.html#BadCipherSuites --- src/core/lib/security/transport/security_connector.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index a7a5d65caf6..b208c78cac4 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -79,8 +79,7 @@ void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb) { /* Defines the cipher suites that we accept by default. All these cipher suites are compliant with HTTP2. */ #define GRPC_SSL_CIPHER_SUITES \ - "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-" \ - "SHA384:ECDHE-RSA-AES256-GCM-SHA384" + "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384" static gpr_once cipher_suites_once = GPR_ONCE_INIT; static const char *cipher_suites = NULL; From 38279ea2e0d6c1704365a1507b9e8d487e5d7837 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ivan=20Vu=C4=8Dica?= Date: Wed, 18 Jan 2017 00:04:16 +0000 Subject: [PATCH 57/60] Bazel rule for building grpc_cli. --- test/cpp/util/BUILD | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/test/cpp/util/BUILD b/test/cpp/util/BUILD index c83f89eb906..9dde22b4d1b 100644 --- a/test/cpp/util/BUILD +++ b/test/cpp/util/BUILD @@ -34,8 +34,8 @@ licenses(["notice"]) # 3-clause BSD cc_binary( name = "testso.so", srcs = [], - deps = ["//:grpc++_unsecure"], linkshared = 1, + deps = ["//:grpc++_unsecure"], ) cc_library( @@ -104,5 +104,29 @@ cc_test( ], ) - - +cc_binary( + name = "grpc_cli", + srcs = [ + "cli_call.cc", + "cli_call.h", + "cli_credentials.cc", + "cli_credentials.h", + "config_grpc_cli.h", + "grpc_cli.cc", + "grpc_tool.cc", + "grpc_tool.h", + "proto_file_parser.cc", + "proto_file_parser.h", + "proto_reflection_descriptor_database.cc", + "proto_reflection_descriptor_database.h", + "service_describer.cc", + "service_describer.h", + "test_config.h", + "test_config_cc.cc", + ], + deps = [ + "//:grpc++", + "//external:gflags", + "//src/proto/grpc/reflection/v1alpha:reflection_proto", + ], +) From 136533e3969f03594cd0700b3e6c1cf915c23e84 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Thu, 27 Apr 2017 14:50:51 +0200 Subject: [PATCH 58/60] fix security_connector.c formatting --- src/core/lib/security/transport/security_connector.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/lib/security/transport/security_connector.c b/src/core/lib/security/transport/security_connector.c index b208c78cac4..30431a4e4a5 100644 --- a/src/core/lib/security/transport/security_connector.c +++ b/src/core/lib/security/transport/security_connector.c @@ -78,7 +78,7 @@ void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb) { /* Defines the cipher suites that we accept by default. All these cipher suites are compliant with HTTP2. */ -#define GRPC_SSL_CIPHER_SUITES \ +#define GRPC_SSL_CIPHER_SUITES \ "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384" static gpr_once cipher_suites_once = GPR_ONCE_INIT; From e37626710719cff0e1c74f0d8b4109743a3f6cbf Mon Sep 17 00:00:00 2001 From: Mehrdad Afshari Date: Wed, 26 Apr 2017 22:28:24 -0700 Subject: [PATCH 59/60] Update macOS install instructions --- INSTALL.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index ac03a5e6ac8..29f0060c81a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -28,16 +28,24 @@ If you plan to build from source and run tests, install the following as well: $ [sudo] apt-get install clang libc++-dev ``` -## Mac OSX +## macOS -For a Mac system, git is not available by default. You will first need to -install Xcode from the Mac AppStore and then run the following command from a -terminal: +On a Mac, you will first need to +install Xcode or +[Command Line Tools for Xcode](https://developer.apple.com/download/more/) +and then run the following command from a terminal: ```sh $ [sudo] xcode-select --install ``` +To build gRPC from source, you may also need to install the following +packages, which you can get from [Homebrew](https://brew.sh): + +```sh + $ brew install autoconf automake libtool shtool +``` + ## Protoc By default gRPC uses [protocol buffers](https://github.com/google/protobuf), From 75d2496ba81271c9bb989c5561851f46fb18d2f0 Mon Sep 17 00:00:00 2001 From: Vijay Pai Date: Thu, 27 Apr 2017 19:05:33 -0700 Subject: [PATCH 60/60] Add gflags to Mac build from source instructions Note that gflags is already listed as something to install under Linux before build from source and running tests but was not included on Mac. --- INSTALL.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 29f0060c81a..6cfa1b6cbaf 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -46,6 +46,11 @@ packages, which you can get from [Homebrew](https://brew.sh): $ brew install autoconf automake libtool shtool ``` +If you plan to build from source and run tests, install the following as well: +```sh + $ brew install gflags +``` + ## Protoc By default gRPC uses [protocol buffers](https://github.com/google/protobuf),