From 29ca79be8939c0386f1c7b17ba66cc3b105c7fc1 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 12:00:08 -0700 Subject: [PATCH 01/43] Command processing, validation --- Makefile | 67 +++- build.yaml | 26 +- test/cpp/util/grpc_cli.cc | 202 ++-------- test/cpp/util/grpc_tool.cc | 369 ++++++++++++++++++ test/cpp/util/grpc_tool.h | 47 +++ test/cpp/util/grpc_tool_test.cc | 219 +++++++++++ tools/run_tests/sources_and_headers.json | 32 ++ tools/run_tests/tests.json | 21 + .../grpc_cli_libs/grpc_cli_libs.vcxproj | 3 + .../grpc_cli_libs.vcxproj.filters | 6 + .../vcxproj/test/grpc_cli/grpc_cli.vcxproj | 6 +- .../grpc_tool_test/grpc_tool_test.vcxproj | 286 ++++++++++++++ .../grpc_tool_test.vcxproj.filters | 232 +++++++++++ 13 files changed, 1332 insertions(+), 184 deletions(-) create mode 100644 test/cpp/util/grpc_tool.cc create mode 100644 test/cpp/util/grpc_tool.h create mode 100644 test/cpp/util/grpc_tool_test.cc create mode 100644 vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj create mode 100644 vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters diff --git a/Makefile b/Makefile index 37999d3b1e2..a1165955a9a 100644 --- a/Makefile +++ b/Makefile @@ -1047,6 +1047,7 @@ grpc_node_plugin: $(BINDIR)/$(CONFIG)/grpc_node_plugin grpc_objective_c_plugin: $(BINDIR)/$(CONFIG)/grpc_objective_c_plugin grpc_python_plugin: $(BINDIR)/$(CONFIG)/grpc_python_plugin grpc_ruby_plugin: $(BINDIR)/$(CONFIG)/grpc_ruby_plugin +grpc_tool_test: $(BINDIR)/$(CONFIG)/grpc_tool_test grpclb_api_test: $(BINDIR)/$(CONFIG)/grpclb_api_test hybrid_end2end_test: $(BINDIR)/$(CONFIG)/hybrid_end2end_test interop_client: $(BINDIR)/$(CONFIG)/interop_client @@ -1402,6 +1403,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/generic_end2end_test \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/hybrid_end2end_test \ $(BINDIR)/$(CONFIG)/interop_client \ @@ -1486,6 +1488,7 @@ buildtests_cxx: privatelibs_cxx \ $(BINDIR)/$(CONFIG)/generic_end2end_test \ $(BINDIR)/$(CONFIG)/golden_file_test \ $(BINDIR)/$(CONFIG)/grpc_cli \ + $(BINDIR)/$(CONFIG)/grpc_tool_test \ $(BINDIR)/$(CONFIG)/grpclb_api_test \ $(BINDIR)/$(CONFIG)/hybrid_end2end_test \ $(BINDIR)/$(CONFIG)/interop_client \ @@ -1772,6 +1775,8 @@ test_cxx: buildtests_cxx $(Q) $(BINDIR)/$(CONFIG)/generic_end2end_test || ( echo test generic_end2end_test failed ; exit 1 ) $(E) "[RUN] Testing golden_file_test" $(Q) $(BINDIR)/$(CONFIG)/golden_file_test || ( echo test golden_file_test failed ; exit 1 ) + $(E) "[RUN] Testing grpc_tool_test" + $(Q) $(BINDIR)/$(CONFIG)/grpc_tool_test || ( echo test grpc_tool_test failed ; exit 1 ) $(E) "[RUN] Testing grpclb_api_test" $(Q) $(BINDIR)/$(CONFIG)/grpclb_api_test || ( echo test grpclb_api_test failed ; exit 1 ) $(E) "[RUN] Testing hybrid_end2end_test" @@ -4089,6 +4094,7 @@ endif LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/cli_call.cc \ + test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ @@ -10998,16 +11004,16 @@ $(BINDIR)/$(CONFIG)/grpc_cli: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a deps_grpc_cli: $(GRPC_CLI_OBJS:.o=.dep) @@ -11204,6 +11210,60 @@ ifneq ($(NO_DEPS),true) endif +GRPC_TOOL_TEST_SRC = \ + $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc \ + $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc \ + test/cpp/util/grpc_tool_test.cc \ + test/cpp/util/string_ref_helper.cc \ + +GRPC_TOOL_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_TOOL_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/grpc_tool_test: openssl_dep_error + +else + + + + +ifeq ($(NO_PROTOBUF),true) + +# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+. + +$(BINDIR)/$(CONFIG)/grpc_tool_test: protobuf_dep_error + +else + +$(BINDIR)/$(CONFIG)/grpc_tool_test: $(PROTOBUF_DEP) $(GRPC_TOOL_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_TOOL_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_tool_test + +endif + +endif + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/echo.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/src/proto/grpc/testing/echo_messages.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_tool_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +$(OBJDIR)/$(CONFIG)/test/cpp/util/string_ref_helper.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_grpc_tool_test: $(GRPC_TOOL_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(GRPC_TOOL_TEST_OBJS:.o=.dep) +endif +endif +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_tool_test.o: $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc +$(OBJDIR)/$(CONFIG)/test/cpp/util/string_ref_helper.o: $(GENDIR)/src/proto/grpc/testing/echo.pb.cc $(GENDIR)/src/proto/grpc/testing/echo.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.pb.cc $(GENDIR)/src/proto/grpc/testing/echo_messages.grpc.pb.cc + + GRPCLB_API_TEST_SRC = \ $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.pb.cc $(GENDIR)/src/proto/grpc/lb/v1/load_balancer.grpc.pb.cc \ test/cpp/grpclb/grpclb_api_test.cc \ @@ -14920,6 +14980,7 @@ test/cpp/util/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/util/byte_buffer_proto_helper.cc: $(OPENSSL_DEP) test/cpp/util/cli_call.cc: $(OPENSSL_DEP) test/cpp/util/create_test_channel.cc: $(OPENSSL_DEP) +test/cpp/util/grpc_tool.cc: $(OPENSSL_DEP) test/cpp/util/proto_file_parser.cc: $(OPENSSL_DEP) test/cpp/util/proto_reflection_descriptor_database.cc: $(OPENSSL_DEP) test/cpp/util/string_ref_helper.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 237394c2056..5b2c5818501 100644 --- a/build.yaml +++ b/build.yaml @@ -1029,10 +1029,12 @@ libs: language: c++ headers: - test/cpp/util/cli_call.h + - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h src: - test/cpp/util/cli_call.cc + - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc deps: @@ -2658,9 +2660,9 @@ targets: - test/cpp/util/grpc_cli.cc deps: - grpc_cli_libs + - grpc++_reflection - grpc++_test_util - grpc_test_util - - grpc++_reflection - grpc++ - grpc - gpr_test_util @@ -2725,6 +2727,28 @@ targets: secure: false vs_config_type: Application vs_project_guid: '{069E9D05-B78B-4751-9252-D21EBAE7DE8E}' +- name: grpc_tool_test + gtest: true + build: test + language: c++ + headers: + - test/cpp/util/string_ref_helper.h + src: + - src/proto/grpc/testing/echo.proto + - src/proto/grpc/testing/echo_messages.proto + - test/cpp/util/grpc_tool_test.cc + - test/cpp/util/string_ref_helper.cc + deps: + - grpc_cli_libs + - grpc++_reflection + - grpc_test_util + - grpc++ + - grpc + - gpr_test_util + - gpr + filegroups: + - grpc++_codegen_proto + - grpc++_config_proto - name: grpclb_api_test gtest: true build: test diff --git a/test/cpp/util/grpc_cli.cc b/test/cpp/util/grpc_cli.cc index 53529da782c..28c87956d67 100644 --- a/test/cpp/util/grpc_cli.cc +++ b/test/cpp/util/grpc_cli.cc @@ -34,21 +34,18 @@ /* A command line tool to talk to a grpc server. Example of talking to grpc interop server: - grpc_cli call localhost:50051 UnaryCall "response_size:10" \ - --protofiles=src/proto/grpc/testing/test.proto --enable_ssl=false + grpc_cli call localhost:50051 UnaryCall src/proto/grpc/testing/test.proto \ + "response_size:10" --enable_ssl=false Options: - 1. --protofiles, use this flag to provide a proto file if the server does - does not have the reflection service. - 2. --proto_path, if your proto file is not under current working directory, + 1. --proto_path, if your proto file is not under current working directory, use this flag to provide a search root. It should work similar to the - counterpart in protoc. This option is valid only when protofiles is - provided. - 3. --metadata specifies metadata to be sent to the server, such as: + counterpart in protoc. + 2. --metadata specifies metadata to be sent to the server, such as: --metadata="MyHeaderKey1:Value1:MyHeaderKey2:Value2" - 4. --enable_ssl, whether to use tls. - 5. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call - 6. --input_binary_file, a file containing the serialized request. The file + 3. --enable_ssl, whether to use tls. + 4. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call + 3. --input_binary_file, a file containing the serialized request. The file can be generated by calling something like: protoc --proto_path=src/proto/grpc/testing/ \ --encode=grpc.testing.SimpleRequest \ @@ -56,7 +53,7 @@ < input.txt > input.bin If this is used and no proto file is provided in the argument list, the method string has to be exact in the form of /package.service/method. - 7. --output_binary_file, a file to write binary format response into, it can + 4. --output_binary_file, a file to write binary format response into, it can be later decoded using protoc: protoc --proto_path=src/proto/grpc/testing/ \ --decode=grpc.testing.SimpleResponse \ @@ -64,182 +61,33 @@ < output.bin > output.txt */ -#include #include +#include #include -#include #include -#include -#include -#include -#include -#include - -#include "test/cpp/util/cli_call.h" -#include "test/cpp/util/proto_file_parser.h" -#include "test/cpp/util/string_ref_helper.h" +#include +#include "test/cpp/util/grpc_tool.h" #include "test/cpp/util/test_config.h" -DEFINE_bool(enable_ssl, true, "Whether to use ssl/tls."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); -DEFINE_string(input_binary_file, "", - "Path to input file containing serialized request."); -DEFINE_string(output_binary_file, "", - "Path to output file to write serialized response."); -DEFINE_string(metadata, "", - "Metadata to send to server, in the form of key1:val1:key2:val2"); -DEFINE_string(proto_path, ".", "Path to look for the proto file."); -// TODO(zyc): support a list of input proto files -DEFINE_string(protofiles, "", "Name of the proto file."); +DEFINE_string(outfile, "", "Output file (default is stdout)"); -void ParseMetadataFlag( - std::multimap* client_metadata) { - if (FLAGS_metadata.empty()) { - return; - } - std::vector fields; - const char* delim = ":"; - size_t cur, next = -1; - do { - cur = next + 1; - next = FLAGS_metadata.find_first_of(delim, cur); - fields.push_back(FLAGS_metadata.substr(cur, next - cur)); - } while (next != grpc::string::npos); - if (fields.size() % 2) { - std::cout << "Failed to parse metadata flag" << std::endl; - exit(1); - } - for (size_t i = 0; i < fields.size(); i += 2) { - client_metadata->insert( - std::pair(fields[i], fields[i + 1])); - } -} - -template -void PrintMetadata(const T& m, const grpc::string& message) { - if (m.empty()) { - return; - } - std::cout << message << std::endl; - grpc::string pair; - for (typename T::const_iterator iter = m.begin(); iter != m.end(); ++iter) { - pair.clear(); - pair.append(iter->first.data(), iter->first.size()); - pair.append(" : "); - pair.append(iter->second.data(), iter->second.size()); - std::cout << pair << std::endl; +static bool SimplePrint(const grpc::string& outfile, + const grpc::string& output) { + if (outfile.empty()) { + std::cout << output; + } else { + std::ofstream output_file(outfile, std::ios::trunc | std::ios::binary); + output_file << output; + output_file.close(); } + return true; } int main(int argc, char** argv) { grpc::testing::InitTest(&argc, &argv, true); - if (argc < 4 || grpc::string(argv[1]) != "call") { - std::cout << "Usage: grpc_cli call server_host:port method_name " - << "[proto file] [text format request] []" << std::endl; - return 1; - } - - grpc::string request_text; - grpc::string server_address(argv[2]); - grpc::string method_name(argv[3]); - std::unique_ptr parser; - grpc::string serialized_request_proto; - - if (argc == 5) { - request_text = argv[4]; - } - - std::shared_ptr creds; - if (!FLAGS_enable_ssl) { - creds = grpc::InsecureChannelCredentials(); - } else { - if (FLAGS_use_auth) { - creds = grpc::GoogleDefaultCredentials(); - } else { - creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); - } - } - std::shared_ptr channel = - grpc::CreateChannel(server_address, creds); - - if (request_text.empty() && FLAGS_input_binary_file.empty()) { - if (isatty(STDIN_FILENO)) { - std::cout << "reading request message from stdin..." << std::endl; - } - std::stringstream input_stream; - input_stream << std::cin.rdbuf(); - request_text = input_stream.str(); - } - - if (!request_text.empty()) { - if (!FLAGS_protofiles.empty()) { - parser.reset(new grpc::testing::ProtoFileParser( - FLAGS_proto_path, FLAGS_protofiles, method_name)); - } else { - parser.reset(new grpc::testing::ProtoFileParser(channel, method_name)); - } - method_name = parser->GetFullMethodName(); - if (parser->HasError()) { - return 1; - } - - if (!FLAGS_input_binary_file.empty()) { - std::cout - << "warning: request given in argv, ignoring --input_binary_file" - << std::endl; - } - } - - if (parser) { - serialized_request_proto = - parser->GetSerializedProto(request_text, true /* is_request */); - if (parser->HasError()) { - return 1; - } - } else if (!FLAGS_input_binary_file.empty()) { - std::ifstream input_file(FLAGS_input_binary_file, - std::ios::in | std::ios::binary); - std::stringstream input_stream; - input_stream << input_file.rdbuf(); - serialized_request_proto = input_stream.str(); - } - std::cout << "connecting to " << server_address << std::endl; - - grpc::string serialized_response_proto; - std::multimap client_metadata; - std::multimap server_initial_metadata, - server_trailing_metadata; - ParseMetadataFlag(&client_metadata); - PrintMetadata(client_metadata, "Sending client initial metadata:"); - grpc::Status s = grpc::testing::CliCall::Call( - channel, method_name, serialized_request_proto, - &serialized_response_proto, client_metadata, &server_initial_metadata, - &server_trailing_metadata); - PrintMetadata(server_initial_metadata, - "Received initial metadata from server:"); - PrintMetadata(server_trailing_metadata, - "Received trailing metadata from server:"); - if (s.ok()) { - std::cout << "Rpc succeeded with OK status" << std::endl; - if (parser) { - grpc::string response_text = parser->GetTextFormat( - serialized_response_proto, false /* is_request */); - if (parser->HasError()) { - return 1; - } - std::cout << "Response: \n " << response_text << std::endl; - } - if (!FLAGS_output_binary_file.empty()) { - std::ofstream output_file(FLAGS_output_binary_file, - std::ios::trunc | std::ios::binary); - output_file << serialized_response_proto; - } - } else { - std::cout << "Rpc failed with status code " << s.error_code() - << ", error message: " << s.error_message() << std::endl; - } - - return 0; + return grpc::testing::GrpcToolMainLib( + argc, (const char**)argv, + std::bind(SimplePrint, FLAGS_outfile, std::placeholders::_1)); } diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc new file mode 100644 index 00000000000..38ac9e55a03 --- /dev/null +++ b/test/cpp/util/grpc_tool.cc @@ -0,0 +1,369 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "grpc_tool.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include "test/cpp/util/cli_call.h" + +#include "test/cpp/util/proto_file_parser.h" +#include "test/cpp/util/proto_reflection_descriptor_database.h" +#include "test/cpp/util/string_ref_helper.h" +#include "test/cpp/util/test_config.h" + +DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); +DEFINE_bool(use_auth, false, "Whether to create default google credentials."); +DEFINE_string(input_binary_file, "", + "Path to input file containing serialized request."); +DEFINE_string(output_binary_file, "", + "Path to output file to write serialized response."); +DEFINE_string(metadata, "", + "Metadata to send to server, in the form of key1:val1:key2:val2"); +DEFINE_string(proto_path, ".", "Path to look for the proto file."); +// TODO(zyc): support a list of input proto files +DEFINE_string(protofiles, "", "Name of the proto file."); + +namespace grpc { +namespace testing { +namespace { + +class GrpcTool { + public: + explicit GrpcTool(); + virtual ~GrpcTool() {} + bool Help(int argc, const char** argv, OutputCallback callback); + bool CallMethod(int argc, const char** argv, OutputCallback callback); + void SetPrintCommandMode(int exit_status) { + print_command_usage_ = true; + usage_exit_status_ = exit_status; + } + + private: + void CommandUsage(const grpc::string& usage) const; + bool print_command_usage_; + int usage_exit_status_; +}; + +template +std::function BindWith4Args( + T&& func) { + return std::bind(std::forward(func), std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3, + std::placeholders::_4); +} + +template +size_t ArraySize(T& a) { + return ((sizeof(a) / sizeof(*(a))) / + static_cast(!(sizeof(a) % sizeof(*(a))))); +} + +void ParseMetadataFlag( + std::multimap* client_metadata) { + if (FLAGS_metadata.empty()) { + return; + } + std::vector fields; + const char* delim = ":"; + size_t cur, next = -1; + do { + cur = next + 1; + next = FLAGS_metadata.find_first_of(delim, cur); + fields.push_back(FLAGS_metadata.substr(cur, next - cur)); + } while (next != grpc::string::npos); + if (fields.size() % 2) { + fprintf(stderr, "Failed to parse metadata flag.\n"); + exit(1); + } + for (size_t i = 0; i < fields.size(); i += 2) { + client_metadata->insert( + std::pair(fields[i], fields[i + 1])); + } +} + +template +void PrintMetadata(const T& m, const grpc::string& message) { + if (m.empty()) { + return; + } + fprintf(stderr, "%s\n", message.c_str()); + grpc::string pair; + for (typename T::const_iterator iter = m.begin(); iter != m.end(); ++iter) { + pair.clear(); + pair.append(iter->first.data(), iter->first.size()); + pair.append(" : "); + pair.append(iter->second.data(), iter->second.size()); + fprintf(stderr, "%s\n", pair.c_str()); + } +} + +struct Command { + const char* command; + std::function function; + int min_args; + int max_args; +}; + +const Command ops[] = { + {"help", BindWith4Args(&GrpcTool::Help), 0, INT_MAX}, + // {"ls", BindWith4Args(&GrpcTool::ListServices), 1, 3}, + // {"list", BindWith4Args(&GrpcTool::ListServices), 1, 3}, + {"call", BindWith4Args(&GrpcTool::CallMethod), 2, 3}, + // {"type", BindWith4Args(&GrpcTool::PrintType), 2, 2}, + // {"parse", BindWith4Args(&GrpcTool::ParseMessage), 2, 3}, + // {"totext", BindWith4Args(&GrpcTool::ToText), 2, 3}, + // {"tobinary", BindWith4Args(&GrpcTool::ToBinary), 2, 3}, +}; + +void Usage(const grpc::string& msg) { + fprintf( + stderr, + "%s\n" + // " grpc_cli ls ... ; List services\n" + " grpc_cli call ... ; Call method\n" + // " grpc_cli type ... ; Print type\n" + // " grpc_cli parse ... ; Parse message\n" + // " grpc_cli totext ... ; Convert binary message to text\n" + // " grpc_cli tobinary ... ; Convert text message to binary\n" + " grpc_cli help ... ; Print this message, or per-command usage\n" + "\n", + msg.c_str()); + + exit(1); +} + +const Command* FindCommand(const grpc::string& name) { + for (int i = 0; i < (int)ArraySize(ops); i++) { + if (name == ops[i].command) { + return &ops[i]; + } + } + return NULL; +} +} // namespace + +int GrpcToolMainLib(int argc, const char** argv, OutputCallback callback) { + if (argc < 2) { + Usage("No command specified"); + } + + grpc::string command = argv[1]; + argc -= 2; + argv += 2; + + const Command* cmd = FindCommand(command); + if (cmd != NULL) { + GrpcTool grpc_tool; + if (argc < cmd->min_args || argc > cmd->max_args) { + // Force the command to print its usage message + fprintf(stderr, "\nWrong number of arguments for %s\n", command.c_str()); + grpc_tool.SetPrintCommandMode(1); + return cmd->function(&grpc_tool, -1, NULL, callback); + } + const bool ok = cmd->function(&grpc_tool, argc, argv, callback); + return ok ? 0 : 1; + } else { + Usage("Invalid command '" + grpc::string(command.c_str()) + "'"); + } + return 1; +} + +GrpcTool::GrpcTool() : print_command_usage_(false), usage_exit_status_(0) {} + +void GrpcTool::CommandUsage(const grpc::string& usage) const { + if (print_command_usage_) { + fprintf(stderr, "\n%s%s\n", usage.c_str(), + (usage.empty() || usage[usage.size() - 1] != '\n') ? "\n" : ""); + exit(usage_exit_status_); + } +} + +bool GrpcTool::Help(int argc, const char** argv, OutputCallback callback) { + CommandUsage( + "Print help\n" + " grpc_cli help [subcommand]\n"); + + if (argc == 0) { + Usage(""); + } else { + const Command* cmd = FindCommand(argv[0]); + if (cmd == NULL) { + Usage("Unknown command '" + grpc::string(argv[0]) + "'"); + } + SetPrintCommandMode(0); + cmd->function(this, -1, NULL, callback); + } + return true; +} + +bool GrpcTool::CallMethod(int argc, const char** argv, + OutputCallback callback) { + CommandUsage( + "Call method\n" + " grpc_cli call
[.] \n" + "
; host:port\n" + " ; Exported service name\n" + " ; Method name\n" + " ; Text protobuffer (overrides infile)\n" + " --protofiles ; Comma separated proto files used as a" + " fallback when parsing request/response\n" + " --proto_path ; The search path of proto files, valid" + " only when --protofiles is given\n" + " --metadata ; The metadata to be sent to the server\n" + " --enable_ssl ; Set whether to use tls\n" + " --use_auth ; Set whether to create default google" + " credentials\n" + " --outfile ; Output filename (defaults to stdout)\n" + " --input_binary_file ; Path to input file in binary format\n" + " --binary_output ; Path to output file in binary format\n"); + + std::stringstream output_ss; + grpc::string request_text; + grpc::string server_address(argv[0]); + grpc::string method_name(argv[1]); + std::unique_ptr parser; + grpc::string serialized_request_proto; + + if (argc == 3) { + request_text = argv[2]; + } + + std::shared_ptr creds; + if (!FLAGS_enable_ssl) { + creds = grpc::InsecureChannelCredentials(); + } else { + if (FLAGS_use_auth) { + creds = grpc::GoogleDefaultCredentials(); + } else { + creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); + } + } + std::shared_ptr channel = + grpc::CreateChannel(server_address, creds); + + if (request_text.empty() && FLAGS_input_binary_file.empty()) { + if (isatty(STDIN_FILENO)) { + std::cout << "reading request message from stdin..." << std::endl; + } + std::stringstream input_stream; + input_stream << std::cin.rdbuf(); + request_text = input_stream.str(); + } + + if (!request_text.empty()) { + if (!FLAGS_protofiles.empty()) { + parser.reset(new grpc::testing::ProtoFileParser( + FLAGS_proto_path, FLAGS_protofiles, method_name)); + } else { + parser.reset(new grpc::testing::ProtoFileParser(channel, method_name)); + } + method_name = parser->GetFullMethodName(); + if (parser->HasError()) { + return 1; + } + + if (!FLAGS_input_binary_file.empty()) { + std::cout + << "warning: request given in argv, ignoring --input_binary_file" + << std::endl; + } + } + + if (parser) { + serialized_request_proto = + parser->GetSerializedProto(request_text, true /* is_request */); + if (parser->HasError()) { + return 1; + } + } else if (!FLAGS_input_binary_file.empty()) { + std::ifstream input_file(FLAGS_input_binary_file, + std::ios::in | std::ios::binary); + std::stringstream input_stream; + input_stream << input_file.rdbuf(); + serialized_request_proto = input_stream.str(); + } + std::cout << "connecting to " << server_address << std::endl; + + grpc::string serialized_response_proto; + std::multimap client_metadata; + std::multimap server_initial_metadata, + server_trailing_metadata; + ParseMetadataFlag(&client_metadata); + PrintMetadata(client_metadata, "Sending client initial metadata:"); + grpc::Status s = grpc::testing::CliCall::Call( + channel, method_name, serialized_request_proto, + &serialized_response_proto, client_metadata, &server_initial_metadata, + &server_trailing_metadata); + PrintMetadata(server_initial_metadata, + "Received initial metadata from server:"); + PrintMetadata(server_trailing_metadata, + "Received trailing metadata from server:"); + if (s.ok()) { + std::cout << "Rpc succeeded with OK status" << std::endl; + if (parser) { + grpc::string response_text = parser->GetTextFormat( + serialized_response_proto, false /* is_request */); + if (parser->HasError()) { + return false; + } + output_ss << "Response: \n " << response_text << std::endl; + } + if (!FLAGS_output_binary_file.empty()) { + std::ofstream output_file(FLAGS_output_binary_file, + std::ios::trunc | std::ios::binary); + output_file << serialized_response_proto; + } + } else { + std::cout << "Rpc failed with status code " << s.error_code() + << ", error message: " << s.error_message() << std::endl; + } + + return callback(output_ss.str()); +} + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h new file mode 100644 index 00000000000..d779737cd5c --- /dev/null +++ b/test/cpp/util/grpc_tool.h @@ -0,0 +1,47 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include + +namespace grpc { +namespace testing { + +typedef std::function OutputCallback; + +int GrpcToolMainLib(int argc, const char **argv, OutputCallback callback); + +} // namespace testing +} // namespace grpc diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc new file mode 100644 index 00000000000..77c3f3fc24d --- /dev/null +++ b/test/cpp/util/grpc_tool_test.cc @@ -0,0 +1,219 @@ +/* + * + * Copyright 2015, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/cpp/util/grpc_tool.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/proto/grpc/testing/echo.grpc.pb.h" +#include "src/proto/grpc/testing/echo.pb.h" +#include "test/core/util/port.h" +#include "test/core/util/test_config.h" +#include "test/cpp/util/string_ref_helper.h" + +using grpc::testing::EchoRequest; +using grpc::testing::EchoResponse; + +namespace grpc { +namespace testing { + +class TestServiceImpl : public ::grpc::testing::EchoTestService::Service { + public: + Status Echo(ServerContext* context, const EchoRequest* request, + EchoResponse* response) GRPC_OVERRIDE { + if (!context->client_metadata().empty()) { + for (std::multimap::const_iterator + iter = context->client_metadata().begin(); + iter != context->client_metadata().end(); ++iter) { + context->AddInitialMetadata(ToString(iter->first), + ToString(iter->second)); + } + } + context->AddTrailingMetadata("trailing_key", "trailing_value"); + response->set_message(request->message()); + return Status::OK; + } +}; + +class GrpcToolTest : public ::testing::Test { + protected: + GrpcToolTest() {} + + void SetUp() GRPC_OVERRIDE { + int port = grpc_pick_unused_port_or_die(); + server_address_ << "localhost:" << port; + // Setup server + ServerBuilder builder; + builder.AddListeningPort(server_address_.str(), + InsecureServerCredentials()); + builder.RegisterService(&service_); + server_ = builder.BuildAndStart(); + } + + void TearDown() GRPC_OVERRIDE { server_->Shutdown(); } + + void ResetStub() { + channel_ = + CreateChannel(server_address_.str(), InsecureChannelCredentials()); + stub_ = grpc::testing::EchoTestService::NewStub(channel_); + } + + std::shared_ptr channel_; + std::unique_ptr stub_; + std::unique_ptr server_; + std::ostringstream server_address_; + TestServiceImpl service_; + reflection::ProtoServerReflectionPlugin plugin_; +}; + +static bool PrintStream(std::stringstream* ss, const grpc::string& output) { + (*ss) << output << std::endl; + return true; +} + +template +static size_t ArraySize(T& a) { + return ((sizeof(a) / sizeof(*(a))) / + static_cast(!(sizeof(a) % sizeof(*(a))))); +} + +#define USAGE_REGEX "( grpc_cli .+\n){2,10}" + +TEST_F(GrpcToolTest, NoCommand) { + // Test input "grpc_cli" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli"}; + // Exit with 1, print usage instruction in stderr + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), "No command specified\n" USAGE_REGEX); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, InvalidCommand) { + // Test input "grpc_cli" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli", "abc"}; + // Exit with 1, print usage instruction in stderr + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), "Invalid command 'abc'\n" USAGE_REGEX); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, HelpCommand) { + // Test input "grpc_cli help" + std::stringstream output_stream; + const char* argv[] = {"grpc_cli", "help"}; + // Exit with 1, print usage instruction in stderr + EXPECT_EXIT(GrpcToolMainLib(ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, + std::placeholders::_1)), + ::testing::ExitedWithCode(1), USAGE_REGEX); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, CallCommand) { + // Test input "grpc_cli call Echo" + std::stringstream output_stream; + grpc::string server_address = server_address_.str(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", + "message: 'Hello'"}; + + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, + std::placeholders::_1))); + // Expected output: "message: \"Hello\"" + EXPECT_TRUE(NULL != + strstr(output_stream.str().c_str(), "message: \"Hello\"")); +} + +TEST_F(GrpcToolTest, TooFewArguments) { + // Test input "grpc_cli call localhost: Echo "message: 'Hello'" + std::stringstream output_stream; + grpc::string server_address = server_address_.str(); + const char* argv[] = {"grpc_cli", "call", "Echo"}; + + // Exit with 1 + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), ".*Wrong number of arguments for call.*"); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +TEST_F(GrpcToolTest, TooManyArguments) { + // Test input "grpc_cli call localhost: Echo Echo "message: 'Hello'" + std::stringstream output_stream; + grpc::string server_address = server_address_.str(); + const char* argv[] = {"grpc_cli", "call", server_address.c_str(), + "Echo", "Echo", "message: 'Hello'"}; + + // Exit with 1 + EXPECT_EXIT( + GrpcToolMainLib( + ArraySize(argv), argv, + std::bind(PrintStream, &output_stream, std::placeholders::_1)), + ::testing::ExitedWithCode(1), ".*Wrong number of arguments for call.*"); + // No output + EXPECT_TRUE(0 == output_stream.tellp()); +} + +} // namespace testing +} // namespace grpc + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + ::testing::InitGoogleTest(&argc, argv); + ::testing::FLAGS_gtest_death_test_style = "threadsafe"; + return RUN_ALL_TESTS(); +} diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 4a27a1d8752..f7232a24f32 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2226,6 +2226,35 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc++", + "grpc++_codegen_proto", + "grpc++_config_proto", + "grpc++_reflection", + "grpc_cli_libs", + "grpc_test_util" + ], + "headers": [ + "src/proto/grpc/testing/echo.grpc.pb.h", + "src/proto/grpc/testing/echo.pb.h", + "src/proto/grpc/testing/echo_messages.grpc.pb.h", + "src/proto/grpc/testing/echo_messages.pb.h", + "test/cpp/util/string_ref_helper.h" + ], + "language": "c++", + "name": "grpc_tool_test", + "src": [ + "test/cpp/util/grpc_tool_test.cc", + "test/cpp/util/string_ref_helper.cc", + "test/cpp/util/string_ref_helper.h" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "grpc", @@ -4466,6 +4495,7 @@ ], "headers": [ "test/cpp/util/cli_call.h", + "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.h" ], @@ -4474,6 +4504,8 @@ "src": [ "test/cpp/util/cli_call.cc", "test/cpp/util/cli_call.h", + "test/cpp/util/grpc_tool.cc", + "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.cc", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.cc", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index d94301b946b..379833bde5e 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -2269,6 +2269,27 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "gtest": true, + "language": "c++", + "name": "grpc_tool_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index d25c692e3e8..3269bab56fa 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -148,12 +148,15 @@ + + + diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index 4add8ed5e13..cbce2f23120 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -4,6 +4,9 @@ test\cpp\util + + test\cpp\util + test\cpp\util @@ -15,6 +18,9 @@ test\cpp\util + + test\cpp\util + test\cpp\util diff --git a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj index 9c8cdc54c25..f9dbe1dcb69 100644 --- a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj +++ b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj @@ -167,15 +167,15 @@ {86E35862-43E8-F59E-F906-AFE0348AD3D2} + + {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} + {0BE77741-552A-929B-A497-4EF7ECE17A64} {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} - - {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} diff --git a/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj new file mode 100644 index 00000000000..c6f65aa30b8 --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj @@ -0,0 +1,286 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {F00D82D4-E988-6D2F-F0B9-9E82BCC2A2B2} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + + + grpc_tool_test + static + Debug + static + Debug + + + grpc_tool_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {86E35862-43E8-F59E-F906-AFE0348AD3D2} + + + {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters new file mode 100644 index 00000000000..731eb2e6ffb --- /dev/null +++ b/vsprojects/vcxproj/test/grpc_tool_test/grpc_tool_test.vcxproj.filters @@ -0,0 +1,232 @@ + + + + + src\proto\grpc\testing + + + src\proto\grpc\testing + + + test\cpp\util + + + test\cpp\util + + + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen\security + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc\impl\codegen + + + include\grpc++\impl\codegen + + + + + test\cpp\util + + + + + + {89fed779-17c5-23da-c8a2-9e868ff34480} + + + {96e4a1a8-0b91-1a6d-ae4d-ddf33abb93c0} + + + {1d9dcc6f-7c1b-cdc3-4c35-73d5968dfd92} + + + {5eca7690-973a-c8ed-84d6-5325f8de43ac} + + + {5789073e-5b84-0ec9-af06-47866647874d} + + + {d3f3293f-204f-7771-fcdf-de673f6b06b6} + + + {7e90f37b-f9cc-0725-b2c1-12aa7d4809ba} + + + {7e4b71ef-8125-6446-bfc1-9bc90beed59c} + + + {169774bd-5c6c-6827-66a4-326b4aef44d6} + + + {1b609b37-ef2a-e5eb-e1ba-ad9e79c77438} + + + {cd1e35d8-8a61-62fe-6ce1-c8936872d1ef} + + + {f7ee4df5-1f47-1e7f-c91e-350382c1b729} + + + {f2166b83-6b0b-d53b-b58b-627bd9efcad2} + + + {bbe36cbc-7fbe-2817-0bd0-d03726f323e6} + + + {e106cd7b-cfa0-0645-f1a9-2acedc23afe7} + + + + From 28263275f0ab22d5a9d01f7e5afddbcc24a9d22c Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 14:38:07 -0700 Subject: [PATCH 02/43] Remove unnecessary dependencies --- Makefile | 7 +++--- build.yaml | 6 ++--- test/cpp/util/grpc_cli.cc | 23 +++++++++++-------- test/cpp/util/grpc_tool.cc | 22 ++++++++++-------- test/cpp/util/grpc_tool.h | 9 ++++---- tools/run_tests/sources_and_headers.json | 13 ++++++----- .../grpc_cli_libs/grpc_cli_libs.vcxproj | 6 +++++ .../grpc_cli_libs.vcxproj.filters | 6 +++++ .../vcxproj/test/grpc_cli/grpc_cli.vcxproj | 9 -------- 9 files changed, 56 insertions(+), 45 deletions(-) diff --git a/Makefile b/Makefile index a1165955a9a..f3a9b2bc7a8 100644 --- a/Makefile +++ b/Makefile @@ -4097,6 +4097,7 @@ LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ + test/cpp/util/string_ref_helper.cc \ PUBLIC_HEADERS_CXX += \ @@ -11004,16 +11005,16 @@ $(BINDIR)/$(CONFIG)/grpc_cli: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(BINDIR)/$(CONFIG)/grpc_cli: $(PROTOBUF_DEP) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(E) "[LD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli + $(Q) $(LDXX) $(LDFLAGS) $(GRPC_CLI_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/grpc_cli endif endif -$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a +$(OBJDIR)/$(CONFIG)/test/cpp/util/grpc_cli.o: $(LIBDIR)/$(CONFIG)/libgrpc_cli_libs.a $(LIBDIR)/$(CONFIG)/libgrpc++_reflection.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a deps_grpc_cli: $(GRPC_CLI_OBJS:.o=.dep) diff --git a/build.yaml b/build.yaml index 5b2c5818501..afad1e113c5 100644 --- a/build.yaml +++ b/build.yaml @@ -1032,15 +1032,18 @@ libs: - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h + - test/cpp/util/string_ref_helper.h src: - test/cpp/util/cli_call.cc - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc + - test/cpp/util/string_ref_helper.cc deps: - grpc++_reflection - grpc++ - grpc_plugin_support + - grpc++_test_config - name: grpc_plugin_support build: protoc language: c++ @@ -2661,11 +2664,8 @@ targets: deps: - grpc_cli_libs - grpc++_reflection - - grpc++_test_util - - grpc_test_util - grpc++ - grpc - - gpr_test_util - gpr - grpc++_test_config - name: grpc_cpp_plugin diff --git a/test/cpp/util/grpc_cli.cc b/test/cpp/util/grpc_cli.cc index 28c87956d67..897ac5ce4a0 100644 --- a/test/cpp/util/grpc_cli.cc +++ b/test/cpp/util/grpc_cli.cc @@ -33,19 +33,24 @@ /* A command line tool to talk to a grpc server. + Run `grpc_cli help` command to see its usage information. + Example of talking to grpc interop server: - grpc_cli call localhost:50051 UnaryCall src/proto/grpc/testing/test.proto \ - "response_size:10" --enable_ssl=false + grpc_cli call localhost:50051 UnaryCall "response_size:10" \ + --protofiles=src/proto/grpc/testing/test.proto --enable_ssl=false Options: - 1. --proto_path, if your proto file is not under current working directory, + 1. --protofiles, use this flag to provide a proto file if the server does + does not have the reflection service. + 2. --proto_path, if your proto file is not under current working directory, use this flag to provide a search root. It should work similar to the - counterpart in protoc. - 2. --metadata specifies metadata to be sent to the server, such as: + counterpart in protoc. This option is valid only when protofiles is + provided. + 3. --metadata specifies metadata to be sent to the server, such as: --metadata="MyHeaderKey1:Value1:MyHeaderKey2:Value2" - 3. --enable_ssl, whether to use tls. - 4. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call - 3. --input_binary_file, a file containing the serialized request. The file + 4. --enable_ssl, whether to use tls. + 5. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call + 6. --input_binary_file, a file containing the serialized request. The file can be generated by calling something like: protoc --proto_path=src/proto/grpc/testing/ \ --encode=grpc.testing.SimpleRequest \ @@ -53,7 +58,7 @@ < input.txt > input.bin If this is used and no proto file is provided in the argument list, the method string has to be exact in the form of /package.service/method. - 4. --output_binary_file, a file to write binary format response into, it can + 7. --output_binary_file, a file to write binary format response into, it can be later decoded using protoc: protoc --proto_path=src/proto/grpc/testing/ \ --decode=grpc.testing.SimpleResponse \ diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 38ac9e55a03..5b3f565ff50 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -31,7 +31,7 @@ * */ -#include "grpc_tool.h" +#include "test/cpp/util/grpc_tool.h" #include #include @@ -47,7 +47,6 @@ #include #include #include -#include #include "test/cpp/util/cli_call.h" #include "test/cpp/util/proto_file_parser.h" @@ -75,8 +74,8 @@ class GrpcTool { public: explicit GrpcTool(); virtual ~GrpcTool() {} - bool Help(int argc, const char** argv, OutputCallback callback); - bool CallMethod(int argc, const char** argv, OutputCallback callback); + bool Help(int argc, const char** argv, GrpcToolOutputCallback callback); + bool CallMethod(int argc, const char** argv, GrpcToolOutputCallback callback); void SetPrintCommandMode(int exit_status) { print_command_usage_ = true; usage_exit_status_ = exit_status; @@ -89,8 +88,8 @@ class GrpcTool { }; template -std::function BindWith4Args( - T&& func) { +std::function +BindWith4Args(T&& func) { return std::bind(std::forward(func), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, std::placeholders::_4); @@ -143,7 +142,8 @@ void PrintMetadata(const T& m, const grpc::string& message) { struct Command { const char* command; - std::function function; + std::function + function; int min_args; int max_args; }; @@ -186,7 +186,8 @@ const Command* FindCommand(const grpc::string& name) { } } // namespace -int GrpcToolMainLib(int argc, const char** argv, OutputCallback callback) { +int GrpcToolMainLib(int argc, const char** argv, + GrpcToolOutputCallback callback) { if (argc < 2) { Usage("No command specified"); } @@ -222,7 +223,8 @@ void GrpcTool::CommandUsage(const grpc::string& usage) const { } } -bool GrpcTool::Help(int argc, const char** argv, OutputCallback callback) { +bool GrpcTool::Help(int argc, const char** argv, + GrpcToolOutputCallback callback) { CommandUsage( "Print help\n" " grpc_cli help [subcommand]\n"); @@ -241,7 +243,7 @@ bool GrpcTool::Help(int argc, const char** argv, OutputCallback callback) { } bool GrpcTool::CallMethod(int argc, const char** argv, - OutputCallback callback) { + GrpcToolOutputCallback callback) { CommandUsage( "Call method\n" " grpc_cli call
[.] \n" diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h index d779737cd5c..9e61abf6418 100644 --- a/test/cpp/util/grpc_tool.h +++ b/test/cpp/util/grpc_tool.h @@ -31,17 +31,16 @@ * */ +#include #include -#include - -#include namespace grpc { namespace testing { -typedef std::function OutputCallback; +typedef std::function GrpcToolOutputCallback; -int GrpcToolMainLib(int argc, const char **argv, OutputCallback callback); +int GrpcToolMainLib(int argc, const char **argv, + GrpcToolOutputCallback callback); } // namespace testing } // namespace grpc diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index f7232a24f32..e2d2796184f 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -2130,14 +2130,11 @@ { "deps": [ "gpr", - "gpr_test_util", "grpc", "grpc++", "grpc++_reflection", "grpc++_test_config", - "grpc++_test_util", - "grpc_cli_libs", - "grpc_test_util" + "grpc_cli_libs" ], "headers": [], "language": "c++", @@ -4491,13 +4488,15 @@ "deps": [ "grpc++", "grpc++_reflection", + "grpc++_test_config", "grpc_plugin_support" ], "headers": [ "test/cpp/util/cli_call.h", "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", - "test/cpp/util/proto_reflection_descriptor_database.h" + "test/cpp/util/proto_reflection_descriptor_database.h", + "test/cpp/util/string_ref_helper.h" ], "language": "c++", "name": "grpc_cli_libs", @@ -4509,7 +4508,9 @@ "test/cpp/util/proto_file_parser.cc", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.cc", - "test/cpp/util/proto_reflection_descriptor_database.h" + "test/cpp/util/proto_reflection_descriptor_database.h", + "test/cpp/util/string_ref_helper.cc", + "test/cpp/util/string_ref_helper.h" ], "third_party": false, "type": "lib" diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index 3269bab56fa..59b2923b037 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -151,6 +151,7 @@ + @@ -161,6 +162,8 @@ + + @@ -172,6 +175,9 @@ {B6E81D84-2ACB-41B8-8781-493A944C7817} + + {3F7D093D-11F9-C4BC-BEB7-18EB28E3F290} + diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index cbce2f23120..d400f8eccf2 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -13,6 +13,9 @@ test\cpp\util + + test\cpp\util + @@ -27,6 +30,9 @@ test\cpp\util + + test\cpp\util + diff --git a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj index f9dbe1dcb69..78a0a63b5d7 100644 --- a/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj +++ b/vsprojects/vcxproj/test/grpc_cli/grpc_cli.vcxproj @@ -170,21 +170,12 @@ {5F575402-3F89-5D1A-6910-9DB8BF5D2BAB} - - {0BE77741-552A-929B-A497-4EF7ECE17A64} - - - {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} - {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} {29D16885-7228-4C31-81ED-5F9187C7F2A9} - - {EAB0A629-17A9-44DB-B5FF-E91A721FE037} - {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} From 724a4e253a579f44d6b3d4ab9599f771837d7580 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 14:56:06 -0700 Subject: [PATCH 03/43] Add header guard --- test/cpp/util/grpc_tool.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h index 9e61abf6418..0c7217d962a 100644 --- a/test/cpp/util/grpc_tool.h +++ b/test/cpp/util/grpc_tool.h @@ -31,6 +31,9 @@ * */ +#ifndef GRPC_TEST_CPP_UTIL_GRPC_TOOL_H +#define GRPC_TEST_CPP_UTIL_GRPC_TOOL_H + #include #include @@ -44,3 +47,5 @@ int GrpcToolMainLib(int argc, const char **argv, } // namespace testing } // namespace grpc + +#endif // GRPC_TEST_CPP_UTIL_GRPC_TOOL_H From 6b86080912d347efd336d9ebba61439cd7950ed6 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 25 Jul 2016 16:10:35 -0700 Subject: [PATCH 04/43] fixing issues from cronet end to end testing changed core logic for executing multiple ops. added call cancellation logic simplified the code. --- .../cronet/transport/cronet_transport.c | 373 +++++++++++++----- 1 file changed, 264 insertions(+), 109 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 25d8aca2508..d589d750be3 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -65,10 +65,11 @@ typedef struct grpc_cronet_transport grpc_cronet_transport; enum send_state { CRONET_SEND_IDLE = 0, - CRONET_REQ_STARTED, CRONET_SEND_HEADER, - CRONET_WRITE, + CRONET_WRITE_PENDING, CRONET_WRITE_COMPLETED, + CRONET_WAIT_FOR_CANCEL, + CRONET_STREAM_CLOSED, }; enum recv_state { @@ -91,13 +92,14 @@ enum e_caller { }; enum callback_id { - CB_SEND_INITIAL_METADATA = 0, - CB_SEND_MESSAGE, - CB_SEND_TRAILING_METADATA, - CB_RECV_MESSAGE, - CB_RECV_INITIAL_METADATA, - CB_RECV_TRAILING_METADATA, - CB_NUM_CALLBACKS + OP_SEND_INITIAL_METADATA = 0, + OP_SEND_MESSAGE, + OP_SEND_TRAILING_METADATA, + OP_RECV_MESSAGE, + OP_RECV_INITIAL_METADATA, + OP_RECV_TRAILING_METADATA, + OP_CANCEL_ERROR, + OP_NUM_CALLBACKS }; struct stream_obj { @@ -117,23 +119,29 @@ struct stream_obj { // Hold the URL char *url; - bool response_headers_received; - bool read_requested; - bool response_trailers_received; + // One bit per operation + bool op_requested[OP_NUM_CALLBACKS]; + bool op_done[OP_NUM_CALLBACKS]; + // Set to true when server indicates no more data will be sent bool read_closed; // Recv message stuff grpc_byte_buffer **recv_message; // Initial metadata stuff grpc_metadata_batch *recv_initial_metadata; + grpc_chttp2_incoming_metadata_buffer initial_metadata; // Trailing metadata stuff grpc_metadata_batch *recv_trailing_metadata; grpc_chttp2_incoming_metadata_buffer imb; + bool imb_valid; // true if there are any valid entries in imb. // This mutex protects receive state machine execution gpr_mu recv_mu; - // we can queue up up to 2 callbacks for each OP - grpc_closure *callback_list[CB_NUM_CALLBACKS][2]; + + // Callbacks to be called when operations complete + grpc_closure *cb_recv_initial_metadata_ready; + grpc_closure *cb_recv_message_ready; + grpc_closure *on_complete; // storage for header cronet_bidirectional_stream_header *headers; @@ -156,34 +164,63 @@ static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_pollset_set *pollset_set) {} -static void enqueue_callbacks(grpc_closure *callback_list[]) { - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - if (callback_list[0]) { - grpc_exec_ctx_sched(&exec_ctx, callback_list[0], GRPC_ERROR_NONE, NULL); - callback_list[0] = NULL; +// Client creates a bunch of operations and invokes "call_start_batch" +// call_start_batch creates a stream_op structure. this structure has info +// needed for executing all the ops. It has on_complete callback that needs +// to be called when all ops are executed. This function keeps track of all +// outstanding operations. It returns true if all operations that were part of +// the stream_op have been completed. +static bool is_op_complete(stream_obj *s) { + int i; + // Check if any requested op is pending + for (i = 0; i < OP_NUM_CALLBACKS; i++) { + if (s->op_requested[i] && !s->op_done[i]) { + gpr_log(GPR_DEBUG, "is_op_complete is FALSE because of %d", i); + return false; + } } - if (callback_list[1]) { - grpc_exec_ctx_sched(&exec_ctx, callback_list[1], GRPC_ERROR_NONE, NULL); - callback_list[1] = NULL; + // Clear the requested/done bits and return true + for (i = 0; i < OP_NUM_CALLBACKS; i++) { + s->op_requested[i] = s->op_done[i] = false; } + return true; +} + +static void enqueue_callback(grpc_closure *callback) { + GPR_ASSERT(callback); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_exec_ctx_sched(&exec_ctx, callback, GRPC_ERROR_NONE, NULL); grpc_exec_ctx_finish(&exec_ctx); } static void on_canceled(cronet_bidirectional_stream *stream) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_canceled %p", stream); + gpr_log(GPR_DEBUG, "on_canceled(%p)", stream); } + stream_obj *s = (stream_obj *)stream->annotation; + s->op_done[OP_CANCEL_ERROR] = true; + + // Terminate any read callback + if (s->cb_recv_message_ready) { + enqueue_callback(s->cb_recv_message_ready); + s->cb_recv_message_ready = 0; + s->op_done[OP_RECV_MESSAGE] = true; + } + // Don't wait to get any trailing metadata + s->op_done[OP_RECV_TRAILING_METADATA] = true; + + next_send_step(s); } static void on_failed(cronet_bidirectional_stream *stream, int net_error) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_failed %p, error = %d", stream, net_error); + gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); } } static void on_succeeded(cronet_bidirectional_stream *stream) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_succeeded %p", stream); + gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); } } @@ -191,31 +228,38 @@ static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received"); + gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, + trailers); } stream_obj *s = (stream_obj *)stream->annotation; memset(&s->imb, 0, sizeof(s->imb)); + s->imb_valid = false; grpc_chttp2_incoming_metadata_buffer_init(&s->imb); unsigned int i = 0; for (i = 0; i < trailers->count; i++) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, + trailers->headers[i].value); + } + grpc_chttp2_incoming_metadata_buffer_add( &s->imb, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(trailers->headers[i].key), grpc_mdstr_from_string(trailers->headers[i].value))); + s->imb_valid = true; } - s->response_trailers_received = true; + s->op_done[OP_RECV_TRAILING_METADATA] = true; next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); } static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_write_completed"); + gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); } stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_MESSAGE]); - s->cronet_send_state = CRONET_WRITE_COMPLETED; + s->op_done[OP_SEND_MESSAGE] = true; next_send_step(s); } @@ -245,14 +289,14 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_read_completed count=%d, total=%d, remaining=%d", - count, s->total_read_bytes, s->remaining_read_bytes); + gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); } if (count > 0) { GPR_ASSERT(s->recv_message); s->remaining_read_bytes -= count; next_recv_step(s, ON_READ_COMPLETE); } else { + gpr_log(GPR_DEBUG, "read_closed = true"); s->read_closed = true; next_recv_step(s, ON_READ_COMPLETE); } @@ -263,21 +307,39 @@ static void on_response_headers_received( const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received"); + gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, + headers, negotiated_protocol); } stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_RECV_INITIAL_METADATA]); - s->response_headers_received = true; + + memset(&s->initial_metadata, 0, sizeof(s->initial_metadata)); + grpc_chttp2_incoming_metadata_buffer_init(&s->initial_metadata); + unsigned int i = 0; + for (i = 0; i < headers->count; i++) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "header key=%s, value=%s", headers->headers[i].key, + headers->headers[i].value); + } + grpc_chttp2_incoming_metadata_buffer_add( + &s->initial_metadata, + grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(headers->headers[i].key), + grpc_mdstr_from_string(headers->headers[i].value))); + } + + grpc_chttp2_incoming_metadata_buffer_publish(&s->initial_metadata, + s->recv_initial_metadata); + enqueue_callback(s->cb_recv_initial_metadata_ready); + s->op_done[OP_RECV_INITIAL_METADATA] = true; next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); } static void on_request_headers_sent(cronet_bidirectional_stream *stream) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent"); + gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); } stream_obj *s = (stream_obj *)stream->annotation; - enqueue_callbacks(s->callback_list[CB_SEND_INITIAL_METADATA]); - s->cronet_send_state = CRONET_SEND_HEADER; + s->op_done[OP_SEND_INITIAL_METADATA] = true; next_send_step(s); } @@ -293,11 +355,13 @@ static cronet_bidirectional_stream_callback callbacks = { on_canceled}; static void invoke_closing_callback(stream_obj *s) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, - s->recv_trailing_metadata); - if (s->callback_list[CB_RECV_TRAILING_METADATA]) { - enqueue_callbacks(s->callback_list[CB_RECV_TRAILING_METADATA]); + if (!is_op_complete(s)) return; + + if (s->imb_valid) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, + s->recv_trailing_metadata); } + enqueue_callback(s->on_complete); } static void set_recv_state(stream_obj *s, enum recv_state state) { @@ -309,29 +373,35 @@ static void set_recv_state(stream_obj *s, enum recv_state state) { // This is invoked from perform_stream_op, and all on_xxxx callbacks. static void next_recv_step(stream_obj *s, enum e_caller caller) { + // gpr_log(GPR_DEBUG, "locking mutex %p", &s->recv_mu); gpr_mu_lock(&s->recv_mu); switch (s->cronet_recv_state) { case CRONET_RECV_IDLE: if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE"); + gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE, caller=%d", + caller); } if (caller == PERFORM_STREAM_OP || caller == ON_RESPONSE_HEADERS_RECEIVED) { - if (s->read_closed && s->response_trailers_received) { - invoke_closing_callback(s); + if (s->read_closed && s->op_done[OP_RECV_TRAILING_METADATA]) { set_recv_state(s, CRONET_RECV_CLOSED); - } else if (s->response_headers_received == true && - s->read_requested == true) { + } else if (s->op_done[OP_RECV_INITIAL_METADATA] == true && + s->op_requested[OP_RECV_MESSAGE]) { set_recv_state(s, CRONET_RECV_READ_LENGTH); s->total_read_bytes = s->remaining_read_bytes = GRPC_HEADER_SIZE_IN_BYTES; GPR_ASSERT(s->read_buffer); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", + s->cbs, s->read_buffer, s->remaining_read_bytes); } cronet_bidirectional_stream_read(s->cbs, s->read_buffer, s->remaining_read_bytes); } + } else if (caller == ON_RESPONSE_TRAILERS_RECEIVED) { + // We get here when we receive trailers directly, i.e. without + // going through a data read operation. + set_recv_state(s, CRONET_RECV_CLOSED); } break; case CRONET_RECV_READ_LENGTH: @@ -340,8 +410,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { } if (caller == ON_READ_COMPLETE) { if (s->read_closed) { - invoke_closing_callback(s); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); + enqueue_callback(s->cb_recv_message_ready); + s->op_done[OP_RECV_MESSAGE] = true; set_recv_state(s, CRONET_RECV_CLOSED); } else { GPR_ASSERT(s->remaining_read_bytes == 0); @@ -352,7 +422,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); GPR_ASSERT(s->read_buffer); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); + gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", + s->cbs, s->read_buffer, s->remaining_read_bytes); } if (s->remaining_read_bytes > 0) { cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, @@ -361,8 +432,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { // Calling the closing callback directly since this is a 0 byte read // for an empty message. process_recv_message(s, NULL); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); - invoke_closing_callback(s); + enqueue_callback(s->cb_recv_message_ready); + s->op_done[OP_RECV_MESSAGE] = true; set_recv_state(s, CRONET_RECV_CLOSED); } } @@ -386,7 +457,8 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { uint8_t *p = (uint8_t *)s->read_buffer; process_recv_message(s, p); set_recv_state(s, CRONET_RECV_IDLE); - enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]); + enqueue_callback(s->cb_recv_message_ready); + s->op_done[OP_RECV_MESSAGE] = true; } } break; @@ -396,7 +468,9 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) { GPR_ASSERT(0); // Should not reach here break; } + invoke_closing_callback(s); gpr_mu_unlock(&s->recv_mu); + // gpr_log(GPR_DEBUG, "unlocking mutex %p", &s->recv_mu); } // This function takes the data from s->write_slice_buffer and assembles into @@ -417,27 +491,69 @@ static void create_grpc_frame(stream_obj *s) { // append actual data memcpy(p, raw_data, length); } - -static void do_write(stream_obj *s) { +// Return false if there is no data to write +static bool do_write(stream_obj *s) { gpr_slice_buffer *sb = &s->write_slice_buffer; GPR_ASSERT(sb->count <= 1); if (sb->count > 0) { create_grpc_frame(s); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write(%p,%p,%d,%d)", + s->cbs, s->write_buffer, (int)s->write_buffer_size, false); } cronet_bidirectional_stream_write(s->cbs, s->write_buffer, (int)s->write_buffer_size, false); + return true; + } else { + return false; + } +} + +static bool init_cronet_stream(stream_obj *s, grpc_transport *gt) { + GPR_ASSERT(s->cbs == NULL); + grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; + GPR_ASSERT(ct->engine); + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); + } + s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); + GPR_ASSERT(s->cbs); + s->read_closed = false; + + for (int i = 0; i < OP_NUM_CALLBACKS; i++) { + s->op_requested[i] = s->op_done[i] = false; + } + s->cronet_send_state = CRONET_SEND_IDLE; + s->cronet_recv_state = CRONET_RECV_IDLE; +} + +static bool do_close_connection(stream_obj *s) { + s->op_done[OP_SEND_TRAILING_METADATA] = true; + if (s->cbs) { + // Send an "empty" write to the far end to signal that we're done. + // This will induce the server to send down trailers. + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write length 0"); + } + cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); + return true; + } else { + // We never created a stream. This was probably an empty request. + invoke_closing_callback(s); + return true; } + return false; } // static void next_send_step(stream_obj *s) { + gpr_log(GPR_DEBUG, "next_send_step cronet_send_state=%d", + s->cronet_send_state); switch (s->cronet_send_state) { case CRONET_SEND_IDLE: GPR_ASSERT( s->cbs); // cronet_bidirectional_stream is not initialized yet. - s->cronet_send_state = CRONET_REQ_STARTED; + s->cronet_send_state = CRONET_SEND_HEADER; if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); } @@ -447,11 +563,51 @@ static void next_send_step(stream_obj *s) { gpr_free(s->header_array.headers); break; case CRONET_SEND_HEADER: - do_write(s); - s->cronet_send_state = CRONET_WRITE; + if (s->op_requested[OP_CANCEL_ERROR]) { + cronet_bidirectional_stream_cancel(s->cbs); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; + } else if (do_write(s) == false && + s->op_requested[OP_SEND_TRAILING_METADATA]) { + if (do_close_connection(s)) { + s->cronet_send_state = CRONET_STREAM_CLOSED; + } + } else { + s->cronet_send_state = CRONET_WRITE_PENDING; + } + break; + case CRONET_WRITE_PENDING: + if (s->op_requested[OP_CANCEL_ERROR]) { + cronet_bidirectional_stream_cancel(s->cbs); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; + } else if (do_write(s) == false && + s->op_requested[OP_SEND_TRAILING_METADATA]) { + if (do_close_connection(s)) { + s->cronet_send_state = CRONET_STREAM_CLOSED; + } + } else { + s->cronet_send_state = CRONET_WRITE_COMPLETED; + } break; case CRONET_WRITE_COMPLETED: - do_write(s); + if (s->op_requested[OP_CANCEL_ERROR]) { + cronet_bidirectional_stream_cancel(s->cbs); + gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; + } else if (do_write(s) == false && + s->op_requested[OP_SEND_TRAILING_METADATA]) { + if (do_close_connection(s)) { + s->cronet_send_state = CRONET_STREAM_CLOSED; + } + } + break; + case CRONET_STREAM_CLOSED: + s->cronet_send_state = CRONET_SEND_IDLE; + break; + case CRONET_WAIT_FOR_CANCEL: + invoke_closing_callback(s); + s->cronet_send_state = CRONET_SEND_IDLE; break; default: GPR_ASSERT(0); @@ -493,7 +649,7 @@ static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, // Create URL by appending :path value to the hostname gpr_asprintf(&s->url, "https://%s%s", host, value); if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); + // gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); } continue; } @@ -511,6 +667,13 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; GPR_ASSERT(ct->engine); stream_obj *s = (stream_obj *)gs; + // Initialize a cronet bidirectional stream if it doesn't exist. + if (s->cbs == NULL) { + init_cronet_stream(s, gt); + } + + s->on_complete = op->on_complete; + if (op->recv_trailing_metadata) { if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, @@ -518,8 +681,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, op->on_complete); } s->recv_trailing_metadata = op->recv_trailing_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_TRAILING_METADATA][0]); - s->callback_list[CB_RECV_TRAILING_METADATA][0] = op->on_complete; + s->op_requested[OP_RECV_TRAILING_METADATA] = true; } if (op->recv_message) { if (grpc_cronet_trace) { @@ -527,24 +689,19 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, op->on_complete); } s->recv_message = (grpc_byte_buffer **)op->recv_message; - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_MESSAGE][1]); - s->callback_list[CB_RECV_MESSAGE][0] = op->recv_message_ready; - s->callback_list[CB_RECV_MESSAGE][1] = op->on_complete; - s->read_requested = true; - next_recv_step(s, PERFORM_STREAM_OP); + s->cb_recv_message_ready = op->recv_message_ready; + s->op_requested[OP_RECV_MESSAGE] = true; } if (op->recv_initial_metadata) { if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_initial_metadata:=%p", - op->on_complete); + gpr_log(GPR_DEBUG, + "perform_stream_op - recv_initial_metadata on_complete=%p, " + "on_ready=%p", + op->on_complete, op->recv_initial_metadata_ready); } s->recv_initial_metadata = op->recv_initial_metadata; - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][0]); - GPR_ASSERT(!s->callback_list[CB_RECV_INITIAL_METADATA][1]); - s->callback_list[CB_RECV_INITIAL_METADATA][0] = - op->recv_initial_metadata_ready; - s->callback_list[CB_RECV_INITIAL_METADATA][1] = op->on_complete; + s->cb_recv_initial_metadata_ready = op->recv_initial_metadata_ready; + s->op_requested[OP_RECV_INITIAL_METADATA] = true; } if (op->send_initial_metadata) { if (grpc_cronet_trace) { @@ -558,8 +715,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, s->header_array.count = s->num_headers; s->header_array.capacity = s->num_headers; s->header_array.headers = s->headers; - GPR_ASSERT(!s->callback_list[CB_SEND_INITIAL_METADATA][0]); - s->callback_list[CB_SEND_INITIAL_METADATA][0] = op->on_complete; + s->op_requested[OP_SEND_INITIAL_METADATA] = true; } if (op->send_message) { if (grpc_cronet_trace) { @@ -572,21 +728,7 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, // TODO (makdharma): add compression support GPR_ASSERT(op->send_message->flags == 0); gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); - if (s->cbs == NULL) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); - } - s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); - GPR_ASSERT(s->cbs); - s->read_closed = false; - s->response_trailers_received = false; - s->response_headers_received = false; - s->cronet_send_state = CRONET_SEND_IDLE; - s->cronet_recv_state = CRONET_RECV_IDLE; - } - GPR_ASSERT(!s->callback_list[CB_SEND_MESSAGE][0]); - s->callback_list[CB_SEND_MESSAGE][0] = op->on_complete; - next_send_step(s); + s->op_requested[OP_SEND_MESSAGE] = true; } if (op->send_trailing_metadata) { if (grpc_cronet_trace) { @@ -594,27 +736,24 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, "perform_stream_op - send_trailing_metadata: on_complete=%p", op->on_complete); } - GPR_ASSERT(!s->callback_list[CB_SEND_TRAILING_METADATA][0]); - s->callback_list[CB_SEND_TRAILING_METADATA][0] = op->on_complete; - if (s->cbs) { - // Send an "empty" write to the far end to signal that we're done. - // This will induce the server to send down trailers. - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write"); - } - cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); - } else { - // We never created a stream. This was probably an empty request. - invoke_closing_callback(s); + s->op_requested[OP_SEND_TRAILING_METADATA] = true; + } + if (op->cancel_error) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "perform_stream_op - cancel_error: on_complete=%p", + op->on_complete); } + s->op_requested[OP_CANCEL_ERROR] = true; } + next_send_step(s); + next_recv_step(s, PERFORM_STREAM_OP); } static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, const void *server_data) { stream_obj *s = (stream_obj *)gs; - memset(s->callback_list, 0, sizeof(s->callback_list)); + memset(s, 0, sizeof(stream_obj)); s->cbs = NULL; gpr_mu_init(&s->recv_mu); s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); @@ -636,6 +775,7 @@ static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, gpr_free(s->read_buffer); gpr_free(s->write_buffer); gpr_free(s->url); + gpr_log(GPR_DEBUG, "destroying %p", &s->recv_mu); gpr_mu_destroy(&s->recv_mu); if (and_free_memory) { gpr_free(and_free_memory); @@ -650,13 +790,28 @@ static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { } } +static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "Unimplemented method"); + } + return NULL; +} + +static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_transport_op *op) { + if (grpc_cronet_trace) { + gpr_log(GPR_DEBUG, "Unimplemented method"); + } + return NULL; +} + const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), "cronet_http", init_stream, set_pollset_do_nothing, set_pollset_set_do_nothing, perform_stream_op, - NULL, + perform_op, destroy_stream, destroy_transport, - NULL}; + get_peer}; From e7068c836ee7077c9e61d8e6a354420477b610c5 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 25 Jul 2016 17:30:45 -0700 Subject: [PATCH 05/43] Remove references to string_ref_helper --- Makefile | 1 - build.yaml | 2 -- test/cpp/util/grpc_tool.cc | 1 - tools/run_tests/sources_and_headers.json | 7 ++----- vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj | 3 --- .../vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters | 6 ------ 6 files changed, 2 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 3f01664b41b..d5b2413ac37 100644 --- a/Makefile +++ b/Makefile @@ -4164,7 +4164,6 @@ LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ - test/cpp/util/string_ref_helper.cc \ PUBLIC_HEADERS_CXX += \ diff --git a/build.yaml b/build.yaml index 5eff6f197ca..0937d1f6cdf 100644 --- a/build.yaml +++ b/build.yaml @@ -1045,13 +1045,11 @@ libs: - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h - - test/cpp/util/string_ref_helper.h src: - test/cpp/util/cli_call.cc - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc - - test/cpp/util/string_ref_helper.cc deps: - grpc++_reflection - grpc++ diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 5b3f565ff50..e227e6027dc 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -51,7 +51,6 @@ #include "test/cpp/util/proto_file_parser.h" #include "test/cpp/util/proto_reflection_descriptor_database.h" -#include "test/cpp/util/string_ref_helper.h" #include "test/cpp/util/test_config.h" DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 21a2da9051a..4b5557dc682 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4501,8 +4501,7 @@ "test/cpp/util/cli_call.h", "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", - "test/cpp/util/proto_reflection_descriptor_database.h", - "test/cpp/util/string_ref_helper.h" + "test/cpp/util/proto_reflection_descriptor_database.h" ], "language": "c++", "name": "grpc_cli_libs", @@ -4514,9 +4513,7 @@ "test/cpp/util/proto_file_parser.cc", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.cc", - "test/cpp/util/proto_reflection_descriptor_database.h", - "test/cpp/util/string_ref_helper.cc", - "test/cpp/util/string_ref_helper.h" + "test/cpp/util/proto_reflection_descriptor_database.h" ], "third_party": false, "type": "lib" diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index 59b2923b037..09034dc33ef 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -151,7 +151,6 @@ - @@ -162,8 +161,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index d400f8eccf2..cbce2f23120 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -13,9 +13,6 @@ test\cpp\util - - test\cpp\util - @@ -30,9 +27,6 @@ test\cpp\util - - test\cpp\util - From 636ef6fb63fb6873a302797376674abbf6d2d4d7 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 26 Jul 2016 10:15:26 -0700 Subject: [PATCH 06/43] minor compiler warning fix --- src/core/ext/transport/cronet/transport/cronet_transport.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index d589d750be3..0a079927ea8 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -509,7 +509,7 @@ static bool do_write(stream_obj *s) { } } -static bool init_cronet_stream(stream_obj *s, grpc_transport *gt) { +static void init_cronet_stream(stream_obj *s, grpc_transport *gt) { GPR_ASSERT(s->cbs == NULL); grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; GPR_ASSERT(ct->engine); @@ -802,7 +802,6 @@ static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, if (grpc_cronet_trace) { gpr_log(GPR_DEBUG, "Unimplemented method"); } - return NULL; } const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), From f4046cdcedbe6be6de173139f4217b34dc876fdd Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 26 Jul 2016 12:22:42 -0700 Subject: [PATCH 07/43] Fix thread leak --- test/cpp/util/grpc_tool_test.cc | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 77c3f3fc24d..57eeed7234a 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -79,29 +79,25 @@ class GrpcToolTest : public ::testing::Test { protected: GrpcToolTest() {} - void SetUp() GRPC_OVERRIDE { + // SetUpServer cannot be used with EXPECT_EXIT. grpc_pick_unused_port_or_die() + // uses atexit() to free chosen ports, and it will spawn a new thread in + // resolve_address_posix.c:192 at exit time. + const grpc::string SetUpServer() { + std::ostringstream server_address; int port = grpc_pick_unused_port_or_die(); - server_address_ << "localhost:" << port; + server_address << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address_.str(), + builder.AddListeningPort(server_address.str(), InsecureServerCredentials()); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); + return server_address.str(); } - void TearDown() GRPC_OVERRIDE { server_->Shutdown(); } + void ShutdownServer() { server_->Shutdown(); } - void ResetStub() { - channel_ = - CreateChannel(server_address_.str(), InsecureChannelCredentials()); - stub_ = grpc::testing::EchoTestService::NewStub(channel_); - } - - std::shared_ptr channel_; - std::unique_ptr stub_; std::unique_ptr server_; - std::ostringstream server_address_; TestServiceImpl service_; reflection::ProtoServerReflectionPlugin plugin_; }; @@ -163,7 +159,8 @@ TEST_F(GrpcToolTest, HelpCommand) { TEST_F(GrpcToolTest, CallCommand) { // Test input "grpc_cli call Echo" std::stringstream output_stream; - grpc::string server_address = server_address_.str(); + + const grpc::string server_address = SetUpServer(); const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", "message: 'Hello'"}; @@ -173,12 +170,12 @@ TEST_F(GrpcToolTest, CallCommand) { // Expected output: "message: \"Hello\"" EXPECT_TRUE(NULL != strstr(output_stream.str().c_str(), "message: \"Hello\"")); + ShutdownServer(); } TEST_F(GrpcToolTest, TooFewArguments) { // Test input "grpc_cli call localhost: Echo "message: 'Hello'" std::stringstream output_stream; - grpc::string server_address = server_address_.str(); const char* argv[] = {"grpc_cli", "call", "Echo"}; // Exit with 1 @@ -194,8 +191,7 @@ TEST_F(GrpcToolTest, TooFewArguments) { TEST_F(GrpcToolTest, TooManyArguments) { // Test input "grpc_cli call localhost: Echo Echo "message: 'Hello'" std::stringstream output_stream; - grpc::string server_address = server_address_.str(); - const char* argv[] = {"grpc_cli", "call", server_address.c_str(), + const char* argv[] = {"grpc_cli", "call", "localhost:10000", "Echo", "Echo", "message: 'Hello'"}; // Exit with 1 From 2f7060bad91c4e3f89b5e3c6818edb810b5058ca Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 26 Jul 2016 16:03:28 -0700 Subject: [PATCH 08/43] test harness changes for compiling --- .../CoreCronetEnd2EndTests.m | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m index 58abb492cea..d2181120e93 100644 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m @@ -77,7 +77,7 @@ static grpc_end2end_test_fixture chttp2_create_fixture_secure_fullstack( gpr_malloc(sizeof(fullstack_secure_fixture_data)); memset(&f, 0, sizeof(f)); - gpr_join_host_port(&ffd->localaddr, "localhost", port); + gpr_join_host_port(&ffd->localaddr, "127.0.0.1", port); f.fixture_data = ffd; f.cq = grpc_completion_queue_create(NULL); @@ -124,15 +124,24 @@ static void chttp2_tear_down_secure_fullstack(grpc_end2end_test_fixture *f) { } static void cronet_init_client_simple_ssl_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { + grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { grpc_arg ssl_name_override = {GRPC_ARG_STRING, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, {"foo.test.google.fr"}}; grpc_channel_args *new_client_args = grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1); - [Cronet setHttp2Enabled:YES]; - [Cronet start]; + static bool done = false; + // TODO (makdharma): DO NOT CHECK IN THIS HACK!!! + if (!done) { + done = true; + [Cronet setHttp2Enabled:YES]; + NSURL *url = [[[NSFileManager defaultManager] + URLsForDirectory:NSDocumentDirectory inDomains:NSUserDomainMask] lastObject]; + NSLog(@"Documents directory: %@", url); + [Cronet start]; + [Cronet startNetLogToFile: @"Documents/cronet_netlog.json" logBytes:YES]; + } cronet_engine *cronetEngine = [Cronet getGlobalEngine]; cronet_init_client_secure_fullstack(f, new_client_args, cronetEngine); @@ -236,7 +245,7 @@ static char *roots_filename; } - (void)testBinaryMetadata { - [self testIndividualCase:"binary_metadata"]; + //[self testIndividualCase:"binary_metadata"]; } - (void)testCallCreds { From 68ca3511264c8af56d70f7576ae84dc296e29989 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 26 Jul 2016 16:48:46 -0700 Subject: [PATCH 09/43] Fix sanity issues --- test/cpp/util/grpc_tool_test.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 57eeed7234a..0f3086b4428 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -88,8 +88,7 @@ class GrpcToolTest : public ::testing::Test { server_address << "localhost:" << port; // Setup server ServerBuilder builder; - builder.AddListeningPort(server_address.str(), - InsecureServerCredentials()); + builder.AddListeningPort(server_address.str(), InsecureServerCredentials()); builder.RegisterService(&service_); server_ = builder.BuildAndStart(); return server_address.str(); From ff47bc0daf9c6081506e4ec462208cc990c7cde1 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Fri, 29 Jul 2016 16:55:35 -0700 Subject: [PATCH 10/43] rewrite the core logic to work with end to end tests. --- .../cronet/transport/cronet_transport.c | 992 +++++++----------- 1 file changed, 380 insertions(+), 612 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 0a079927ea8..694d346fc33 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -51,8 +51,41 @@ #define GRPC_HEADER_SIZE_IN_BYTES 5 -// Global flag that gets set with GRPC_TRACE env variable -int grpc_cronet_trace = 1; +enum OP_ID { + OP_SEND_INITIAL_METADATA = 0, + OP_SEND_MESSAGE, + OP_SEND_TRAILING_METADATA, + OP_RECV_MESSAGE, + OP_RECV_INITIAL_METADATA, + OP_RECV_TRAILING_METADATA, + OP_CANCEL_ERROR, + OP_ON_COMPLETE, + OP_NUM_OPS +}; + +/* Cronet callbacks */ + +static void on_request_headers_sent(cronet_bidirectional_stream *); +static void on_response_headers_received(cronet_bidirectional_stream *, + const cronet_bidirectional_stream_header_array *, + const char *); +static void on_write_completed(cronet_bidirectional_stream *, const char *); +static void on_read_completed(cronet_bidirectional_stream *, char *, int); +static void on_response_trailers_received(cronet_bidirectional_stream *, + const cronet_bidirectional_stream_header_array *); +static void on_succeeded(cronet_bidirectional_stream *); +static void on_failed(cronet_bidirectional_stream *, int); +//static void on_canceled(cronet_bidirectional_stream *); +static cronet_bidirectional_stream_callback cronet_callbacks = { + on_request_headers_sent, + on_response_headers_received, + on_read_completed, + on_write_completed, + on_response_trailers_received, + on_succeeded, + on_failed, + NULL //on_canceled +}; // Cronet transport object struct grpc_cronet_transport { @@ -60,428 +93,198 @@ struct grpc_cronet_transport { cronet_engine *engine; char *host; }; - typedef struct grpc_cronet_transport grpc_cronet_transport; -enum send_state { - CRONET_SEND_IDLE = 0, - CRONET_SEND_HEADER, - CRONET_WRITE_PENDING, - CRONET_WRITE_COMPLETED, - CRONET_WAIT_FOR_CANCEL, - CRONET_STREAM_CLOSED, -}; +struct read_state { + // vars to store data coming from cronet + char *read_buffer; + bool length_field_received; + int received_bytes; + int remaining_bytes; + int length_field; + char grpc_header_bytes[GRPC_HEADER_SIZE_IN_BYTES]; + char *payload_field; + + // vars for holding data destined for the application + struct grpc_slice_buffer_stream sbs; + gpr_slice_buffer read_slice_buffer; -enum recv_state { - CRONET_RECV_IDLE = 0, - CRONET_RECV_READ_LENGTH, - CRONET_RECV_READ_DATA, - CRONET_RECV_CLOSED, -}; + // vars for trailing metadata + grpc_chttp2_incoming_metadata_buffer trailing_metadata; + bool trailing_metadata_valid; -static const char *recv_state_name[] = { - "CRONET_RECV_IDLE", "CRONET_RECV_READ_LENGTH", "CRONET_RECV_READ_DATA,", - "CRONET_RECV_CLOSED"}; + // vars for initial metadata + grpc_chttp2_incoming_metadata_buffer initial_metadata; +}; -// Enum that identifies calling function. -enum e_caller { - PERFORM_STREAM_OP, - ON_READ_COMPLETE, - ON_RESPONSE_HEADERS_RECEIVED, - ON_RESPONSE_TRAILERS_RECEIVED +struct write_state { + char *write_buffer; }; -enum callback_id { - OP_SEND_INITIAL_METADATA = 0, - OP_SEND_MESSAGE, - OP_SEND_TRAILING_METADATA, - OP_RECV_MESSAGE, - OP_RECV_INITIAL_METADATA, - OP_RECV_TRAILING_METADATA, - OP_CANCEL_ERROR, - OP_NUM_CALLBACKS +#define MAX_PENDING_OPS 10 +struct op_storage { + grpc_transport_stream_op pending_ops[MAX_PENDING_OPS]; + int wrptr; + int rdptr; + int num_pending_ops; }; struct stream_obj { - // we store received bytes here as they trickle in. - gpr_slice_buffer write_slice_buffer; + grpc_transport_stream_op *curr_op; + grpc_cronet_transport curr_ct; + grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; - gpr_slice slice; - gpr_slice_buffer read_slice_buffer; - struct grpc_slice_buffer_stream sbs; - char *read_buffer; - int remaining_read_bytes; - int total_read_bytes; - - char *write_buffer; - size_t write_buffer_size; - // Hold the URL - char *url; + // TODO (makdharma) : make a sub structure for tracking state + bool state_op_done[OP_NUM_OPS]; + bool state_callback_received[OP_NUM_OPS]; - // One bit per operation - bool op_requested[OP_NUM_CALLBACKS]; - bool op_done[OP_NUM_CALLBACKS]; - // Set to true when server indicates no more data will be sent - bool read_closed; - - // Recv message stuff - grpc_byte_buffer **recv_message; - // Initial metadata stuff - grpc_metadata_batch *recv_initial_metadata; - grpc_chttp2_incoming_metadata_buffer initial_metadata; - // Trailing metadata stuff - grpc_metadata_batch *recv_trailing_metadata; - grpc_chttp2_incoming_metadata_buffer imb; - bool imb_valid; // true if there are any valid entries in imb. - - // This mutex protects receive state machine execution - gpr_mu recv_mu; - - // Callbacks to be called when operations complete - grpc_closure *cb_recv_initial_metadata_ready; - grpc_closure *cb_recv_message_ready; - grpc_closure *on_complete; - - // storage for header - cronet_bidirectional_stream_header *headers; - uint32_t num_headers; - cronet_bidirectional_stream_header_array header_array; - // state tracking - enum recv_state cronet_recv_state; - enum send_state cronet_send_state; + // Read state + struct read_state rs; + // Write state + struct write_state ws; + // OP storage + struct op_storage storage; }; - typedef struct stream_obj stream_obj; -static void next_send_step(stream_obj *s); -static void next_recv_step(stream_obj *s, enum e_caller caller); +/* Globals */ +cronet_bidirectional_stream_header_array header_array; -static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_pollset *pollset) {} - -static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, - grpc_transport *gt, grpc_stream *gs, - grpc_pollset_set *pollset_set) {} - -// Client creates a bunch of operations and invokes "call_start_batch" -// call_start_batch creates a stream_op structure. this structure has info -// needed for executing all the ops. It has on_complete callback that needs -// to be called when all ops are executed. This function keeps track of all -// outstanding operations. It returns true if all operations that were part of -// the stream_op have been completed. -static bool is_op_complete(stream_obj *s) { - int i; - // Check if any requested op is pending - for (i = 0; i < OP_NUM_CALLBACKS; i++) { - if (s->op_requested[i] && !s->op_done[i]) { - gpr_log(GPR_DEBUG, "is_op_complete is FALSE because of %d", i); - return false; - } - } - // Clear the requested/done bits and return true - for (i = 0; i < OP_NUM_CALLBACKS; i++) { - s->op_requested[i] = s->op_done[i] = false; - } - return true; -} - -static void enqueue_callback(grpc_closure *callback) { - GPR_ASSERT(callback); - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_exec_ctx_sched(&exec_ctx, callback, GRPC_ERROR_NONE, NULL); - grpc_exec_ctx_finish(&exec_ctx); +// +static void execute_curr_stream_op(stream_obj *s); + +/************************************************************* + Op Storage +*/ + +static void add_pending_op(struct op_storage *storage, grpc_transport_stream_op *op) { + GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); + storage->num_pending_ops++; + gpr_log(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", + storage->wrptr, storage->num_pending_ops); + memcpy(&storage->pending_ops[storage->wrptr], op, sizeof(grpc_transport_stream_op)); + storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; } -static void on_canceled(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_canceled(%p)", stream); - } - stream_obj *s = (stream_obj *)stream->annotation; - s->op_done[OP_CANCEL_ERROR] = true; - - // Terminate any read callback - if (s->cb_recv_message_ready) { - enqueue_callback(s->cb_recv_message_ready); - s->cb_recv_message_ready = 0; - s->op_done[OP_RECV_MESSAGE] = true; - } - // Don't wait to get any trailing metadata - s->op_done[OP_RECV_TRAILING_METADATA] = true; - - next_send_step(s); +static grpc_transport_stream_op *pop_pending_op(struct op_storage *storage) { + if (storage->num_pending_ops == 0) return NULL; + grpc_transport_stream_op *result = &storage->pending_ops[storage->rdptr]; + storage->rdptr = (storage->rdptr + 1) % MAX_PENDING_OPS; + storage->num_pending_ops--; + gpr_log(GPR_DEBUG, "popping op @rdptr=%d. %d more left in queue", + storage->rdptr, storage->num_pending_ops); + return result; } +/************************************************************* +Cronet Callback Ipmlementation +*/ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); - } + gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); } static void on_succeeded(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); - } -} - -static void on_response_trailers_received( - cronet_bidirectional_stream *stream, - const cronet_bidirectional_stream_header_array *trailers) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, - trailers); - } - stream_obj *s = (stream_obj *)stream->annotation; - - memset(&s->imb, 0, sizeof(s->imb)); - s->imb_valid = false; - grpc_chttp2_incoming_metadata_buffer_init(&s->imb); - unsigned int i = 0; - for (i = 0; i < trailers->count; i++) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, - trailers->headers[i].value); - } - - grpc_chttp2_incoming_metadata_buffer_add( - &s->imb, grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(trailers->headers[i].key), - grpc_mdstr_from_string(trailers->headers[i].value))); - s->imb_valid = true; - } - s->op_done[OP_RECV_TRAILING_METADATA] = true; - next_recv_step(s, ON_RESPONSE_TRAILERS_RECEIVED); + gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); } -static void on_write_completed(cronet_bidirectional_stream *stream, - const char *data) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); - } - stream_obj *s = (stream_obj *)stream->annotation; - s->op_done[OP_SEND_MESSAGE] = true; - next_send_step(s); -} -static void process_recv_message(stream_obj *s, const uint8_t *recv_data) { - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes); - uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - if (s->total_read_bytes > 0) { - // Only copy if there is non-zero number of bytes - memcpy(dst_p, recv_data, (size_t)s->total_read_bytes); - gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice); - } - grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0); - *s->recv_message = (grpc_byte_buffer *)&s->sbs; -} - -static int parse_grpc_header(const uint8_t *data) { - const uint8_t *p = data + 1; - int length = 0; - length |= ((uint8_t)*p++) << 24; - length |= ((uint8_t)*p++) << 16; - length |= ((uint8_t)*p++) << 8; - length |= ((uint8_t)*p++); - return length; -} - -static void on_read_completed(cronet_bidirectional_stream *stream, char *data, - int count) { +static void on_request_headers_sent(cronet_bidirectional_stream *stream) { + gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); - } - if (count > 0) { - GPR_ASSERT(s->recv_message); - s->remaining_read_bytes -= count; - next_recv_step(s, ON_READ_COMPLETE); - } else { - gpr_log(GPR_DEBUG, "read_closed = true"); - s->read_closed = true; - next_recv_step(s, ON_READ_COMPLETE); - } + s->state_op_done[OP_SEND_INITIAL_METADATA] = true; + s->state_callback_received[OP_SEND_INITIAL_METADATA] = true; + execute_curr_stream_op(s); } static void on_response_headers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, + gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, headers, negotiated_protocol); - } - stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->initial_metadata, 0, sizeof(s->initial_metadata)); - grpc_chttp2_incoming_metadata_buffer_init(&s->initial_metadata); + stream_obj *s = (stream_obj *)stream->annotation; + memset(&s->rs.initial_metadata, 0, sizeof(s->rs.initial_metadata)); + grpc_chttp2_incoming_metadata_buffer_init(&s->rs.initial_metadata); unsigned int i = 0; for (i = 0; i < headers->count; i++) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "header key=%s, value=%s", headers->headers[i].key, - headers->headers[i].value); - } grpc_chttp2_incoming_metadata_buffer_add( - &s->initial_metadata, + &s->rs.initial_metadata, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(headers->headers[i].key), grpc_mdstr_from_string(headers->headers[i].value))); } - - grpc_chttp2_incoming_metadata_buffer_publish(&s->initial_metadata, - s->recv_initial_metadata); - enqueue_callback(s->cb_recv_initial_metadata_ready); - s->op_done[OP_RECV_INITIAL_METADATA] = true; - next_recv_step(s, ON_RESPONSE_HEADERS_RECEIVED); + s->state_callback_received[OP_RECV_INITIAL_METADATA] = true; + execute_curr_stream_op(s); } -static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); - } +static void on_write_completed(cronet_bidirectional_stream *stream, + const char *data) { + gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); stream_obj *s = (stream_obj *)stream->annotation; - s->op_done[OP_SEND_INITIAL_METADATA] = true; - next_send_step(s); -} - -// Callback function pointers (invoked by cronet in response to events) -static cronet_bidirectional_stream_callback callbacks = { - on_request_headers_sent, - on_response_headers_received, - on_read_completed, - on_write_completed, - on_response_trailers_received, - on_succeeded, - on_failed, - on_canceled}; - -static void invoke_closing_callback(stream_obj *s) { - if (!is_op_complete(s)) return; - - if (s->imb_valid) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->imb, - s->recv_trailing_metadata); + if (s->ws.write_buffer) { + gpr_free(s->ws.write_buffer); + s->ws.write_buffer = NULL; } - enqueue_callback(s->on_complete); + s->state_callback_received[OP_SEND_MESSAGE] = true; + execute_curr_stream_op(s); } -static void set_recv_state(stream_obj *s, enum recv_state state) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "next_state = %s", recv_state_name[state]); +static void on_read_completed(cronet_bidirectional_stream *stream, char *data, + int count) { + stream_obj *s = (stream_obj *)stream->annotation; + gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + if (count > 0) { + s->rs.received_bytes += count; + s->rs.remaining_bytes -= count; + if (s->rs.remaining_bytes > 0) { + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); + cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer + s->rs.received_bytes, s->rs.remaining_bytes); + } else { + execute_curr_stream_op(s); + } + s->state_callback_received[OP_RECV_MESSAGE] = true; } - s->cronet_recv_state = state; } -// This is invoked from perform_stream_op, and all on_xxxx callbacks. -static void next_recv_step(stream_obj *s, enum e_caller caller) { - // gpr_log(GPR_DEBUG, "locking mutex %p", &s->recv_mu); - gpr_mu_lock(&s->recv_mu); - switch (s->cronet_recv_state) { - case CRONET_RECV_IDLE: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_IDLE, caller=%d", - caller); - } - if (caller == PERFORM_STREAM_OP || - caller == ON_RESPONSE_HEADERS_RECEIVED) { - if (s->read_closed && s->op_done[OP_RECV_TRAILING_METADATA]) { - set_recv_state(s, CRONET_RECV_CLOSED); - } else if (s->op_done[OP_RECV_INITIAL_METADATA] == true && - s->op_requested[OP_RECV_MESSAGE]) { - set_recv_state(s, CRONET_RECV_READ_LENGTH); - s->total_read_bytes = s->remaining_read_bytes = - GRPC_HEADER_SIZE_IN_BYTES; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", - s->cbs, s->read_buffer, s->remaining_read_bytes); - } - cronet_bidirectional_stream_read(s->cbs, s->read_buffer, - s->remaining_read_bytes); - } - } else if (caller == ON_RESPONSE_TRAILERS_RECEIVED) { - // We get here when we receive trailers directly, i.e. without - // going through a data read operation. - set_recv_state(s, CRONET_RECV_CLOSED); - } - break; - case CRONET_RECV_READ_LENGTH: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_LENGTH"); - } - if (caller == ON_READ_COMPLETE) { - if (s->read_closed) { - enqueue_callback(s->cb_recv_message_ready); - s->op_done[OP_RECV_MESSAGE] = true; - set_recv_state(s, CRONET_RECV_CLOSED); - } else { - GPR_ASSERT(s->remaining_read_bytes == 0); - set_recv_state(s, CRONET_RECV_READ_DATA); - s->total_read_bytes = s->remaining_read_bytes = - parse_grpc_header((const uint8_t *)s->read_buffer); - s->read_buffer = - gpr_realloc(s->read_buffer, (uint32_t)s->remaining_read_bytes); - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read(%p,%p,%d)", - s->cbs, s->read_buffer, s->remaining_read_bytes); - } - if (s->remaining_read_bytes > 0) { - cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer, - s->remaining_read_bytes); - } else { - // Calling the closing callback directly since this is a 0 byte read - // for an empty message. - process_recv_message(s, NULL); - enqueue_callback(s->cb_recv_message_ready); - s->op_done[OP_RECV_MESSAGE] = true; - set_recv_state(s, CRONET_RECV_CLOSED); - } - } - } - break; - case CRONET_RECV_READ_DATA: - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_recv_state = CRONET_RECV_READ_DATA"); - } - if (caller == ON_READ_COMPLETE) { - if (s->remaining_read_bytes > 0) { - int offset = s->total_read_bytes - s->remaining_read_bytes; - GPR_ASSERT(s->read_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()"); - } - cronet_bidirectional_stream_read( - s->cbs, (char *)s->read_buffer + offset, s->remaining_read_bytes); - } else { - gpr_slice_buffer_init(&s->read_slice_buffer); - uint8_t *p = (uint8_t *)s->read_buffer; - process_recv_message(s, p); - set_recv_state(s, CRONET_RECV_IDLE); - enqueue_callback(s->cb_recv_message_ready); - s->op_done[OP_RECV_MESSAGE] = true; - } - } - break; - case CRONET_RECV_CLOSED: - break; - default: - GPR_ASSERT(0); // Should not reach here - break; +static void on_response_trailers_received( + cronet_bidirectional_stream *stream, + const cronet_bidirectional_stream_header_array *trailers) { + gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, + trailers); + stream_obj *s = (stream_obj *)stream->annotation; + memset(&s->rs.trailing_metadata, 0, sizeof(s->rs.trailing_metadata)); + s->rs.trailing_metadata_valid = false; + grpc_chttp2_incoming_metadata_buffer_init(&s->rs.trailing_metadata); + unsigned int i = 0; + for (i = 0; i < trailers->count; i++) { + gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, + trailers->headers[i].value); + grpc_chttp2_incoming_metadata_buffer_add( + &s->rs.trailing_metadata, grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(trailers->headers[i].key), + grpc_mdstr_from_string(trailers->headers[i].value))); + s->rs.trailing_metadata_valid = true; } - invoke_closing_callback(s); - gpr_mu_unlock(&s->recv_mu); - // gpr_log(GPR_DEBUG, "unlocking mutex %p", &s->recv_mu); + s->state_callback_received[OP_RECV_TRAILING_METADATA] = true; + execute_curr_stream_op(s); } +/************************************************************* +Utility functions. Can be in their own file +*/ // This function takes the data from s->write_slice_buffer and assembles into // a contiguous byte stream with 5 byte gRPC header prepended. -static void create_grpc_frame(stream_obj *s) { - gpr_slice slice = gpr_slice_buffer_take_first(&s->write_slice_buffer); - uint8_t *raw_data = GPR_SLICE_START_PTR(slice); +static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, + char **pp_write_buffer, int *p_write_buffer_size) { + gpr_slice slice = gpr_slice_buffer_take_first(write_slice_buffer); size_t length = GPR_SLICE_LENGTH(slice); - s->write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; - s->write_buffer = gpr_realloc(s->write_buffer, s->write_buffer_size); - uint8_t *p = (uint8_t *)s->write_buffer; + // TODO (makdharma): FREE THIS!! HACK! + *p_write_buffer_size = (int)length + GRPC_HEADER_SIZE_IN_BYTES; + char *write_buffer = gpr_malloc(length + GRPC_HEADER_SIZE_IN_BYTES); + *pp_write_buffer = write_buffer; + uint8_t *p = (uint8_t *)write_buffer; // Append 5 byte header *p++ = 0; *p++ = (uint8_t)(length >> 24); @@ -489,135 +292,22 @@ static void create_grpc_frame(stream_obj *s) { *p++ = (uint8_t)(length >> 8); *p++ = (uint8_t)(length); // append actual data - memcpy(p, raw_data, length); -} -// Return false if there is no data to write -static bool do_write(stream_obj *s) { - gpr_slice_buffer *sb = &s->write_slice_buffer; - GPR_ASSERT(sb->count <= 1); - if (sb->count > 0) { - create_grpc_frame(s); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write(%p,%p,%d,%d)", - s->cbs, s->write_buffer, (int)s->write_buffer_size, false); - } - cronet_bidirectional_stream_write(s->cbs, s->write_buffer, - (int)s->write_buffer_size, false); - return true; - } else { - return false; - } + memcpy(p, GPR_SLICE_START_PTR(slice), length); } -static void init_cronet_stream(stream_obj *s, grpc_transport *gt) { - GPR_ASSERT(s->cbs == NULL); - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - GPR_ASSERT(ct->engine); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); - } - s->cbs = cronet_bidirectional_stream_create(ct->engine, s, &callbacks); - GPR_ASSERT(s->cbs); - s->read_closed = false; - - for (int i = 0; i < OP_NUM_CALLBACKS; i++) { - s->op_requested[i] = s->op_done[i] = false; - } - s->cronet_send_state = CRONET_SEND_IDLE; - s->cronet_recv_state = CRONET_RECV_IDLE; -} - -static bool do_close_connection(stream_obj *s) { - s->op_done[OP_SEND_TRAILING_METADATA] = true; - if (s->cbs) { - // Send an "empty" write to the far end to signal that we're done. - // This will induce the server to send down trailers. - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_write length 0"); - } - cronet_bidirectional_stream_write(s->cbs, "abc", 0, true); - return true; - } else { - // We never created a stream. This was probably an empty request. - invoke_closing_callback(s); - return true; - } - return false; -} - -// -static void next_send_step(stream_obj *s) { - gpr_log(GPR_DEBUG, "next_send_step cronet_send_state=%d", - s->cronet_send_state); - switch (s->cronet_send_state) { - case CRONET_SEND_IDLE: - GPR_ASSERT( - s->cbs); // cronet_bidirectional_stream is not initialized yet. - s->cronet_send_state = CRONET_SEND_HEADER; - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start to %s", s->url); - } - cronet_bidirectional_stream_start(s->cbs, s->url, 0, "POST", - &s->header_array, false); - // we no longer need the memory that was allocated earlier. - gpr_free(s->header_array.headers); - break; - case CRONET_SEND_HEADER: - if (s->op_requested[OP_CANCEL_ERROR]) { - cronet_bidirectional_stream_cancel(s->cbs); - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; - } else if (do_write(s) == false && - s->op_requested[OP_SEND_TRAILING_METADATA]) { - if (do_close_connection(s)) { - s->cronet_send_state = CRONET_STREAM_CLOSED; - } - } else { - s->cronet_send_state = CRONET_WRITE_PENDING; - } - break; - case CRONET_WRITE_PENDING: - if (s->op_requested[OP_CANCEL_ERROR]) { - cronet_bidirectional_stream_cancel(s->cbs); - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; - } else if (do_write(s) == false && - s->op_requested[OP_SEND_TRAILING_METADATA]) { - if (do_close_connection(s)) { - s->cronet_send_state = CRONET_STREAM_CLOSED; - } - } else { - s->cronet_send_state = CRONET_WRITE_COMPLETED; - } - break; - case CRONET_WRITE_COMPLETED: - if (s->op_requested[OP_CANCEL_ERROR]) { - cronet_bidirectional_stream_cancel(s->cbs); - gpr_log(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - s->cronet_send_state = CRONET_WAIT_FOR_CANCEL; - } else if (do_write(s) == false && - s->op_requested[OP_SEND_TRAILING_METADATA]) { - if (do_close_connection(s)) { - s->cronet_send_state = CRONET_STREAM_CLOSED; - } - } - break; - case CRONET_STREAM_CLOSED: - s->cronet_send_state = CRONET_SEND_IDLE; - break; - case CRONET_WAIT_FOR_CANCEL: - invoke_closing_callback(s); - s->cronet_send_state = CRONET_SEND_IDLE; - break; - default: - GPR_ASSERT(0); - break; - } +static void enqueue_callback(grpc_closure *callback) { + GPR_ASSERT(callback); + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + grpc_exec_ctx_sched(&exec_ctx, callback, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_finish(&exec_ctx); } -static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, - const char *host, - stream_obj *s) { +static void convert_metadata_to_cronet_headers( + grpc_linked_mdelem *head, + const char *host, + char **pp_url, + cronet_bidirectional_stream_header **pp_headers, + size_t *p_num_headers) { grpc_linked_mdelem *curr = head; // Walk the linked list and get number of header fields uint32_t num_headers_available = 0; @@ -625,17 +315,19 @@ static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, curr = curr->next; num_headers_available++; } - // Allocate enough memory - s->headers = (cronet_bidirectional_stream_header *)gpr_malloc( + // Allocate enough memory. TODO (makdharma): FREE MEMORY! HACK HACK + cronet_bidirectional_stream_header *headers = + (cronet_bidirectional_stream_header *)gpr_malloc( sizeof(cronet_bidirectional_stream_header) * num_headers_available); + *pp_headers = headers; // Walk the linked list again, this time copying the header fields. // s->num_headers // can be less than num_headers_available, as some headers are not used for // cronet curr = head; - s->num_headers = 0; - while (s->num_headers < num_headers_available) { + int num_headers = 0; + while (num_headers < num_headers_available) { grpc_mdelem *mdelem = curr->md; curr = curr->next; const char *key = grpc_mdstr_as_c_string(mdelem->key); @@ -647,161 +339,237 @@ static void convert_metadata_to_cronet_headers(grpc_linked_mdelem *head, } if (strcmp(key, ":path") == 0) { // Create URL by appending :path value to the hostname - gpr_asprintf(&s->url, "https://%s%s", host, value); - if (grpc_cronet_trace) { - // gpr_log(GPR_DEBUG, "extracted URL = %s", s->url); - } + gpr_asprintf(pp_url, "https://%s%s", host, value); continue; } - s->headers[s->num_headers].key = key; - s->headers[s->num_headers].value = value; - s->num_headers++; + headers[num_headers].key = key; + headers[num_headers].value = value; + num_headers++; if (curr == NULL) { break; } } + *p_num_headers = num_headers; } -static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, grpc_transport_stream_op *op) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - GPR_ASSERT(ct->engine); - stream_obj *s = (stream_obj *)gs; - // Initialize a cronet bidirectional stream if it doesn't exist. - if (s->cbs == NULL) { - init_cronet_stream(s, gt); - } - - s->on_complete = op->on_complete; +static int parse_grpc_header(const uint8_t *data) { + const uint8_t *p = data + 1; + int length = 0; + length |= ((uint8_t)*p++) << 24; + length |= ((uint8_t)*p++) << 16; + length |= ((uint8_t)*p++) << 8; + length |= ((uint8_t)*p++); + return length; +} - if (op->recv_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - recv_trailing_metadata: on_complete=%p", - op->on_complete); - } - s->recv_trailing_metadata = op->recv_trailing_metadata; - s->op_requested[OP_RECV_TRAILING_METADATA] = true; - } - if (op->recv_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - recv_message: on_complete=%p", - op->on_complete); - } - s->recv_message = (grpc_byte_buffer **)op->recv_message; - s->cb_recv_message_ready = op->recv_message_ready; - s->op_requested[OP_RECV_MESSAGE] = true; - } - if (op->recv_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - recv_initial_metadata on_complete=%p, " - "on_ready=%p", - op->on_complete, op->recv_initial_metadata_ready); - } - s->recv_initial_metadata = op->recv_initial_metadata; - s->cb_recv_initial_metadata_ready = op->recv_initial_metadata_ready; - s->op_requested[OP_RECV_INITIAL_METADATA] = true; - } - if (op->send_initial_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_initial_metadata: on_complete=%p", - op->on_complete); - } - s->num_headers = 0; - convert_metadata_to_cronet_headers(op->send_initial_metadata->list.head, - ct->host, s); - s->header_array.count = s->num_headers; - s->header_array.capacity = s->num_headers; - s->header_array.headers = s->headers; - s->op_requested[OP_SEND_INITIAL_METADATA] = true; +/* +Op Execution +*/ + +static bool op_can_be_run(stream_obj *s, enum OP_ID op_id) { + if (op_id == OP_SEND_INITIAL_METADATA) { + // already executed + if (s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; + } + if (op_id == OP_RECV_INITIAL_METADATA) { + // already executed + if (s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; + // we haven't sent headers yet. + if (!s->state_callback_received[OP_SEND_INITIAL_METADATA]) return false; + // we haven't received headers yet. + if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; + } + if (op_id == OP_SEND_MESSAGE) { + // already executed + if (s->state_op_done[OP_SEND_MESSAGE]) return false; + // we haven't received headers yet. + if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; + } + if (op_id == OP_RECV_MESSAGE) { + // already executed + if (s->state_op_done[OP_RECV_MESSAGE]) return false; + // we haven't received headers yet. + if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; + } + if (op_id == OP_RECV_TRAILING_METADATA) { + // already executed + if (s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + // we haven't received trailers yet. + if (!s->state_callback_received[OP_RECV_TRAILING_METADATA]) return false; + } + if (op_id == OP_SEND_TRAILING_METADATA) { + // already executed + if (s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; + // we haven't sent message yet + if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; + } + + if (op_id == OP_ON_COMPLETE) { + // already executed + if (s->state_op_done[OP_ON_COMPLETE]) return false; + // Check if every op that was asked for is done. + if (s->curr_op->send_initial_metadata && !s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; + if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; + if (s->curr_op->send_trailing_metadata && !s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; + if (s->curr_op->recv_initial_metadata && !s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; + if (s->curr_op->recv_message && !s->state_op_done[OP_RECV_MESSAGE]) return false; + if (s->curr_op->recv_trailing_metadata && !s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; } - if (op->send_message) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - send_message: on_complete=%p", - op->on_complete); - } - grpc_byte_stream_next(exec_ctx, op->send_message, &s->slice, - op->send_message->length, NULL); + return true; +} + +static void execute_curr_stream_op(stream_obj *s) { + if (s->curr_op->send_initial_metadata && op_can_be_run(s, OP_SEND_INITIAL_METADATA)) { + // This OP is the beginning. Reset various states + memset(&s->rs, 0, sizeof(s->rs)); + memset(&s->ws, 0, sizeof(s->ws)); + memset(s->state_op_done, 0, sizeof(s->state_op_done)); + memset(s->state_callback_received, 0, sizeof(s->state_callback_received)); + // Start new cronet stream + GPR_ASSERT(s->cbs == NULL); + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); + s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); + char *url; + convert_metadata_to_cronet_headers(s->curr_op->send_initial_metadata->list.head, + s->curr_ct.host, &url, &header_array.headers, &header_array.count); + header_array.capacity = header_array.count; + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start"); + cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &header_array, false); + s->state_op_done[OP_SEND_INITIAL_METADATA] = true; + } else if (s->curr_op->recv_initial_metadata && + op_can_be_run(s, OP_RECV_INITIAL_METADATA)) { + grpc_chttp2_incoming_metadata_buffer_publish(&s->rs.initial_metadata, + s->curr_op->recv_initial_metadata); + enqueue_callback(s->curr_op->recv_initial_metadata_ready); + s->state_op_done[OP_RECV_INITIAL_METADATA] = true; + // We are ready to execute send_message. + execute_curr_stream_op(s); + } else if (s->curr_op->send_message && op_can_be_run(s, OP_SEND_MESSAGE)) { + // TODO (makdharma): Make into a standalone function + gpr_slice_buffer write_slice_buffer; + gpr_slice slice; + gpr_slice_buffer_init(&write_slice_buffer); + grpc_byte_stream_next(NULL, s->curr_op->send_message, &slice, + s->curr_op->send_message->length, NULL); // Check that compression flag is not ON. We don't support compression yet. // TODO (makdharma): add compression support - GPR_ASSERT(op->send_message->flags == 0); - gpr_slice_buffer_add(&s->write_slice_buffer, s->slice); - s->op_requested[OP_SEND_MESSAGE] = true; - } - if (op->send_trailing_metadata) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, - "perform_stream_op - send_trailing_metadata: on_complete=%p", - op->on_complete); + GPR_ASSERT(s->curr_op->send_message->flags == 0); + gpr_slice_buffer_add(&write_slice_buffer, slice); + GPR_ASSERT(write_slice_buffer.count == 1); // Empty request not handled yet + if (write_slice_buffer.count > 0) { + int write_buffer_size; + create_grpc_frame(&write_slice_buffer, &s->ws.write_buffer, &write_buffer_size); + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", s->ws.write_buffer); + cronet_bidirectional_stream_write(s->cbs, s->ws.write_buffer, + write_buffer_size, false); // TODO: What if this is not the last write? } - s->op_requested[OP_SEND_TRAILING_METADATA] = true; - } - if (op->cancel_error) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "perform_stream_op - cancel_error: on_complete=%p", - op->on_complete); + s->state_op_done[OP_SEND_MESSAGE] = true; + } else if (s->curr_op->recv_message && op_can_be_run(s, OP_RECV_MESSAGE)) { + if (s->rs.length_field_received == false) { + if (s->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && s->rs.remaining_bytes == 0) { + // Start a read operation for data + s->rs.length_field_received = true; + s->rs.length_field = s->rs.remaining_bytes = + parse_grpc_header((const uint8_t *)s->rs.read_buffer); + GPR_ASSERT(s->rs.length_field > 0); // Empty message? + gpr_log(GPR_DEBUG, "length field = %d", s->rs.length_field); + s->rs.read_buffer = gpr_malloc(s->rs.length_field); + GPR_ASSERT(s->rs.read_buffer); + s->rs.remaining_bytes = s->rs.length_field; + s->rs.received_bytes = 0; + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); + cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, + s->rs.remaining_bytes); + } else if (s->rs.remaining_bytes == 0) { + // Start a read operation for first 5 bytes (GRPC header) + s->rs.read_buffer = s->rs.grpc_header_bytes; + s->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; + s->rs.received_bytes = 0; + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); + cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, + s->rs.remaining_bytes); + } + } else if (s->rs.remaining_bytes == 0) { + gpr_log(GPR_DEBUG, "read operation complete"); + gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->rs.length_field); + uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); + memcpy(dst_p, s->rs.read_buffer, (size_t)s->rs.length_field); + gpr_slice_buffer_init(&s->rs.read_slice_buffer); + gpr_slice_buffer_add(&s->rs.read_slice_buffer, read_data_slice); + grpc_slice_buffer_stream_init(&s->rs.sbs, &s->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)s->curr_op->recv_message) = (grpc_byte_buffer *)&s->rs.sbs; + enqueue_callback(s->curr_op->recv_message_ready); + s->state_op_done[OP_RECV_MESSAGE] = true; + execute_curr_stream_op(s); + } + } else if (s->curr_op->recv_trailing_metadata && + op_can_be_run(s, OP_RECV_TRAILING_METADATA)) { + if (s->rs.trailing_metadata_valid) { + grpc_chttp2_incoming_metadata_buffer_publish( + &s->rs.trailing_metadata, s->curr_op->recv_trailing_metadata); + s->rs.trailing_metadata_valid = false; } - s->op_requested[OP_CANCEL_ERROR] = true; + s->state_op_done[OP_RECV_TRAILING_METADATA] = true; + execute_curr_stream_op(s); + } else if (s->curr_op->send_trailing_metadata && + op_can_be_run(s, OP_SEND_TRAILING_METADATA)) { + + gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + cronet_bidirectional_stream_write(s->cbs, "", 0, true); + s->state_op_done[OP_SEND_TRAILING_METADATA] = true; + } else if (op_can_be_run(s, OP_ON_COMPLETE)) { + // All ops are complete. Call the on_complete callback + enqueue_callback(s->curr_op->on_complete); + s->state_op_done[OP_ON_COMPLETE] = true; + cronet_bidirectional_stream_destroy(s->cbs); + s->cbs = NULL; } - next_send_step(s); - next_recv_step(s, PERFORM_STREAM_OP); } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, const void *server_data) { stream_obj *s = (stream_obj *)gs; - memset(s, 0, sizeof(stream_obj)); - s->cbs = NULL; - gpr_mu_init(&s->recv_mu); - s->read_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - s->write_buffer = gpr_malloc(GRPC_HEADER_SIZE_IN_BYTES); - gpr_slice_buffer_init(&s->write_slice_buffer); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "cronet_transport - init_stream"); - } + memset(&s->storage, 0, sizeof(s->storage)); + s->curr_op = NULL; return 0; } -static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy stream"); - } +static void set_pollset_do_nothing(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_pollset *pollset) {} + +static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, + grpc_transport *gt, grpc_stream *gs, + grpc_pollset_set *pollset_set) {} + +static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, grpc_transport_stream_op *op) { + gpr_log(GPR_DEBUG, "perform_stream_op"); stream_obj *s = (stream_obj *)gs; - s->cbs = NULL; - gpr_free(s->read_buffer); - gpr_free(s->write_buffer); - gpr_free(s->url); - gpr_log(GPR_DEBUG, "destroying %p", &s->recv_mu); - gpr_mu_destroy(&s->recv_mu); - if (and_free_memory) { - gpr_free(and_free_memory); + memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); + add_pending_op(&s->storage, op); + if (s->curr_op == NULL) { + s->curr_op = pop_pending_op(&s->storage); } + s->curr_gs = gs; + execute_curr_stream_op(s); +} + +static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, + grpc_stream *gs, void *and_free_memory) { } static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { - grpc_cronet_transport *ct = (grpc_cronet_transport *)gt; - gpr_free(ct->host); - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Destroy transport"); - } } static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Unimplemented method"); - } return NULL; } static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_transport_op *op) { - if (grpc_cronet_trace) { - gpr_log(GPR_DEBUG, "Unimplemented method"); - } } const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), From 4d8283f93bed3b9d35362cad2c4fb38474b547d6 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:11:49 -0700 Subject: [PATCH 11/43] Add thrift module --- .gitmodules | 3 +++ third_party/thrift | 1 + 2 files changed, 4 insertions(+) create mode 160000 third_party/thrift diff --git a/.gitmodules b/.gitmodules index ce647f3c455..3bfd3c9ce10 100644 --- a/.gitmodules +++ b/.gitmodules @@ -17,3 +17,6 @@ [submodule "third_party/nanopb"] path = third_party/nanopb url = https://github.com/nanopb/nanopb.git +[submodule "third_party/thrift"] + path = third_party/thrift + url = https://github.com/apache/thrift.git diff --git a/third_party/thrift b/third_party/thrift new file mode 160000 index 00000000000..bcad91771b7 --- /dev/null +++ b/third_party/thrift @@ -0,0 +1 @@ +Subproject commit bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c From bc618eed71f1c2a2b69351a6aa60e8bb460773b9 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:35:51 -0700 Subject: [PATCH 12/43] thrift serializer --- Makefile | 3 + build.yaml | 9 + .../grpc++/impl/codegen/thrift_serializer.h | 148 +++++++++++++++ .../impl/codegen/thrift_serializer_inl.h | 169 ++++++++++++++++++ include/grpc++/impl/codegen/thrift_utils.h | 90 ++++++++++ tools/run_tests/sources_and_headers.json | 22 ++- .../grpc++_test_util/grpc++_test_util.vcxproj | 3 + .../grpc++_test_util.vcxproj.filters | 9 + 8 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 include/grpc++/impl/codegen/thrift_serializer.h create mode 100644 include/grpc++/impl/codegen/thrift_serializer_inl.h create mode 100644 include/grpc++/impl/codegen/thrift_utils.h diff --git a/Makefile b/Makefile index 8b6114bd7f9..986c25780f2 100644 --- a/Makefile +++ b/Makefile @@ -3923,6 +3923,9 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/time.h \ include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ + include/grpc++/impl/codegen/thrift_serializer.h \ + include/grpc++/impl/codegen/thrift_serializer_inl.h \ + include/grpc++/impl/codegen/thrift_utils.h \ LIBGRPC++_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_UTIL_SRC)))) diff --git a/build.yaml b/build.yaml index 37c95ccdc0a..35af7a6c7a5 100644 --- a/build.yaml +++ b/build.yaml @@ -780,6 +780,14 @@ filegroups: - src/cpp/ext/reflection.pb.cc uses: - grpc++_codegen_proto +- name: thrift_util + language: c++ + public_headers: + - include/grpc++/impl/codegen/thrift_serializer.h + - include/grpc++/impl/codegen/thrift_serializer_inl.h + - include/grpc++/impl/codegen/thrift_utils.h + uses: + - grpc++_codegen_base libs: - name: gpr build: all @@ -1021,6 +1029,7 @@ libs: - grpc++_codegen_base_src - grpc++_codegen_proto - grpc++_config_proto + - thrift_util - name: grpc++_unsecure build: all language: c++ diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h new file mode 100644 index 00000000000..315d76cf2a5 --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -0,0 +1,148 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H + #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H + +#include +#include + +#include +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace util { + +using apache::thrift::protocol::TBinaryProtocolT; +using apache::thrift::protocol::TCompactProtocolT; +using apache::thrift::protocol::TNetworkBigEndian; +using apache::thrift::transport::TMemoryBuffer; +using apache::thrift::transport::TBufferBase; +using apache::thrift::transport::TTransport; +using std::shared_ptr; + + +template +class ThriftSerializer { +public: + ThriftSerializer() + : prepared_ (false) + , lastDeserialized_ (false) + , serializeVersion_ (false) {} + + /** + * Serialize the passed type into the internal buffer + * and returns a pointer to internal buffer and its size + * + */ + template + void serialize(const T& fields, const uint8_t** serializedBuffer, + size_t* serializedLen); + + /** + * Serialize the passed type into the byte buffer + */ + template + void serialize(const T& fields, grpc_byte_buffer** bp); + + /** + * Deserialize the passed char array into the passed type, returns the number + * of bytes that have been consumed from the passed string. + */ + template + uint32_t deserialize(const uint8_t* serializedBuffer, size_t length, + T* fields); + + /** + * Deserialize the passed byte buffer to passed type, returns the number + * of bytes consumed from byte buffer + */ + template + uint32_t deserialize(grpc_byte_buffer* buffer, T* msg); + + void setSerializeVersion(bool value); + + virtual ~ThriftSerializer() {} + + + /** + * Set the container size limit to deserialize + * This function should be called after buffer_ is initialized + */ + void setContainerSizeLimit(int32_t container_limit) { + if (!prepared_) { + prepare(); + } + protocol_->setContainerSizeLimit(container_limit); + } + + /** + * Set the string size limit to deserialize + * This function should be called after buffer_ is initialized + */ + void setStringSizeLimit(int32_t string_limit) { + if (!prepared_) { + prepare(); + } + protocol_->setStringSizeLimit(string_limit); + } + + + private: + void prepare(); + + private: + typedef P Protocol; + bool prepared_; + bool lastDeserialized_; + boost::shared_ptr buffer_; + shared_ptr protocol_; + bool serializeVersion_; +}; // ThriftSerializer + +template +struct ThriftSerializerBinary : public ThriftSerializer > {}; + + +template +struct ThriftSerializerCompact : public ThriftSerializer >{ }; + +}}} // namespace apache::thrift::util + +#include + +#endif diff --git a/include/grpc++/impl/codegen/thrift_serializer_inl.h b/include/grpc++/impl/codegen/thrift_serializer_inl.h new file mode 100644 index 00000000000..866ecf6312d --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_serializer_inl.h @@ -0,0 +1,169 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H + #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H + +#include +#include +#include +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace util { + +using apache::thrift::protocol::TMessageType; + +template +template +void ThriftSerializer::serialize(const T& fields, + const uint8_t** serializedBuffer, size_t* serializedLen) { + + // prepare or reset buffer + if (!prepared_ || lastDeserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + lastDeserialized_ = false; + + // if required serialize protocol version + if (serializeVersion_) { + protocol_->writeMessageBegin("", TMessageType(0), 0); + } + + // serilaize fields into buffer + fields.write(protocol_.get()); + + // write the end of message + if (serializeVersion_) { + protocol_->writeMessageEnd(); + } + + // assign buffer to string + uint8_t* byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + *serializedBuffer = byteBuffer; + *serializedLen = byteBufferSize; +} + +template +template +void ThriftSerializer::serialize(const T& fields, grpc_byte_buffer** bp) { + + const uint8_t* byteBuffer; + size_t byteBufferSize; + serialize(fields, &byteBuffer, &byteBufferSize); + + gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + + *bp = grpc_raw_byte_buffer_create(&slice, 1); + + gpr_slice_unref(slice); +} + +template +template +uint32_t ThriftSerializer::deserialize(const uint8_t* serializedBuffer, + size_t length, T* fields) { + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + lastDeserialized_ = true; + + //reset buffer transport + buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + + // read the protocol version if necessary + if (serializeVersion_) { + std::string name = ""; + TMessageType mt = (TMessageType) 0; + int32_t seq_id = 0; + protocol_->readMessageBegin(name, mt, seq_id); + } + + // deserialize buffer into fields + uint32_t len = fields->read(protocol_.get()); + + // read the end of message + if (serializeVersion_) { + protocol_->readMessageEnd(); + } + + return len; +} + +template +template +uint32_t ThriftSerializer::deserialize(grpc_byte_buffer* bp, T* fields) { + grpc_byte_buffer_reader reader; + grpc_byte_buffer_reader_init(&reader, bp); + + gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); + + uint32_t len = deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), fields); + + gpr_slice_unref(slice); + + grpc_byte_buffer_reader_destroy(&reader); + + return len; +} + +template +void ThriftSerializer::setSerializeVersion(bool value) { + serializeVersion_ = value; +} + +template +void +ThriftSerializer::prepare() +{ + + buffer_.reset(new TMemoryBuffer()); + + // create a protocol for the memory buffer transport + protocol_.reset(new Protocol(buffer_)); + + prepared_ = true; +} + +}}} // namespace apache::thrift::util + +#endif diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h new file mode 100644 index 00000000000..629441149fd --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -0,0 +1,90 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H +#define GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace grpc { + +using apache::thrift::util::ThriftSerializerCompact; + +template +class SerializationTraits::value>::type> { + public: + + static Status Serialize(const T& msg, + grpc_byte_buffer** bp, bool* own_buffer) { + + *own_buffer = true; + + ThriftSerializerCompact serializer; + + serializer.serialize(msg, bp); + + return Status(StatusCode::OK, "ok"); + } + + static Status Deserialize(grpc_byte_buffer* buffer, + T* msg, + int max_message_size) { + if (!buffer) { + return Status(StatusCode::INTERNAL, "No payload"); + } + + ThriftSerializerCompact deserializer; + deserializer.deserialize(buffer, msg); + + grpc_byte_buffer_destroy(buffer); + + return Status(StatusCode::OK, "ok"); + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e85dd2c8200..06ac86fc9c4 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4411,7 +4411,8 @@ "grpc++_codegen_base_src", "grpc++_codegen_proto", "grpc++_config_proto", - "grpc_test_util" + "grpc_test_util", + "thrift_util" ], "headers": [ "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", @@ -6784,5 +6785,24 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [ + "grpc++_codegen_base" + ], + "headers": [ + "include/grpc++/impl/codegen/thrift_serializer.h", + "include/grpc++/impl/codegen/thrift_serializer_inl.h", + "include/grpc++/impl/codegen/thrift_utils.h" + ], + "language": "c++", + "name": "thrift_util", + "src": [ + "include/grpc++/impl/codegen/thrift_serializer.h", + "include/grpc++/impl/codegen/thrift_serializer_inl.h", + "include/grpc++/impl/codegen/thrift_utils.h" + ], + "third_party": false, + "type": "filegroup" } ] diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj index d0fca9ba65a..96ce3b89164 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj @@ -200,6 +200,9 @@ + + + diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters index cc98c604080..2105e672df9 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters @@ -192,6 +192,15 @@ include\grpc++\impl\codegen + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + From dd7a2a35e23042413c232ea9f00a4b7dbbce5648 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 21:57:47 -0700 Subject: [PATCH 13/43] gthrift codegen and Dockerfile --- tools/grift/Dockerfile | 63 + tools/grift/grpc_plugins_generator.patch | 2417 ++++++++++++++++++++++ 2 files changed, 2480 insertions(+) create mode 100644 tools/grift/Dockerfile create mode 100644 tools/grift/grpc_plugins_generator.patch diff --git a/tools/grift/Dockerfile b/tools/grift/Dockerfile new file mode 100644 index 00000000000..5238010ea92 --- /dev/null +++ b/tools/grift/Dockerfile @@ -0,0 +1,63 @@ +# Copyright 2016, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +FROM ubuntu:14.04 + +RUN apt-get update && \ + apt-get install -y \ + git build-essential \ + pkg-config flex \ + bison \ + libkrb5-dev \ + libsasl2-dev \ + libnuma-dev \ + pkg-config \ + libssl-dev \ + autoconf libtool \ + cmake \ + libiberty-dev \ + g++ unzip \ + curl make automake libtool + +# Configure git +RUN git config --global user.name " " && \ + git config --global user.email " " + +RUN git clone https://github.com/grpc/grpc + +RUN cd grpc && git submodule update --init + +RUN cd grpc/third_party/thrift && git am --signoff < ../../tools/grift/grpc_plugins_generator.patch + +RUN cd grpc/third_party/protobuf && ./autogen.sh && ./configure && \ + make -j && make check -j && make install && ldconfig + +RUN cd grpc && make -j && make install + +RUN cd grpc/third_party/thrift && ./bootstrap.sh && ./configure && make -j && make install \ No newline at end of file diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch new file mode 100644 index 00000000000..2779dfb59eb --- /dev/null +++ b/tools/grift/grpc_plugins_generator.patch @@ -0,0 +1,2417 @@ +From 0894590b5020c38106d4ebb2291994668c64f9dd Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Sun, 31 Jul 2016 15:47:47 -0700 +Subject: [PATCH 1/3] don't build tests + +--- + Makefile.am | 7 ++----- + lib/cpp/Makefile.am | 7 ++----- + 2 files changed, 4 insertions(+), 10 deletions(-) + +diff --git a/Makefile.am b/Makefile.am +index 10fe49a..d49caac 100755 +--- a/Makefile.am ++++ b/Makefile.am +@@ -21,10 +21,6 @@ ACLOCAL_AMFLAGS = -I ./aclocal + + SUBDIRS = compiler/cpp lib + +-if WITH_TESTS +-SUBDIRS += test +-endif +- + if WITH_TUTORIAL + SUBDIRS += tutorial + endif +@@ -117,4 +113,5 @@ EXTRA_DIST = \ + CHANGES \ + NOTICE \ + README.md \ +- Thrift.podspec ++ Thrift.podspec \ ++ test +diff --git a/lib/cpp/Makefile.am b/lib/cpp/Makefile.am +index 6fd15d2..7de1fad 100755 +--- a/lib/cpp/Makefile.am ++++ b/lib/cpp/Makefile.am +@@ -27,10 +27,6 @@ moc__%.cpp: %.h + + SUBDIRS = . + +-if WITH_TESTS +-SUBDIRS += test +-endif +- + pkgconfigdir = $(libdir)/pkgconfig + + lib_LTLIBRARIES = libthrift.la +@@ -277,7 +273,8 @@ EXTRA_DIST = \ + thrift-qt.pc.in \ + thrift-qt5.pc.in \ + src/thrift/qt/CMakeLists.txt \ +- $(WINDOWS_DIST) ++ $(WINDOWS_DIST) \ ++ test + + style-local: + $(CPPSTYLE_CMD) +-- +2.8.0.rc3.226.g39d4020 + + +From 04244fa7805740761db757e4c44251f723d85839 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Sun, 31 Jul 2016 16:16:40 -0700 +Subject: [PATCH 2/3] grpc cpp plugins generator with example + +--- + compiler/cpp/src/generate/t_cpp_generator.cc | 476 +++++++++++++++++++++++---- + tutorial/cpp/CMakeLists.txt | 53 --- + tutorial/cpp/CppClient.cpp | 134 ++++---- + tutorial/cpp/CppServer.cpp | 226 ++++--------- + tutorial/cpp/Makefile.am | 58 ++-- + tutorial/cpp/test.thrift | 13 + + 6 files changed, 590 insertions(+), 370 deletions(-) + delete mode 100644 tutorial/cpp/CMakeLists.txt + create mode 100644 tutorial/cpp/test.thrift + +diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc +index 6c04899..9c3399b 100644 +--- a/compiler/cpp/src/generate/t_cpp_generator.cc ++++ b/compiler/cpp/src/generate/t_cpp_generator.cc +@@ -162,6 +162,8 @@ public: + bool specialized = false); + void generate_function_helpers(t_service* tservice, t_function* tfunction); + void generate_service_async_skeleton(t_service* tservice); ++ void generate_service_stub_interface(t_service* tservice); ++ void generate_service_stub(t_service* tservice); + + /** + * Serialization constructs +@@ -883,10 +885,10 @@ void t_cpp_generator::generate_struct_declaration(ofstream& out, + bool is_user_struct) { + string extends = ""; + if (is_exception) { +- extends = " : public ::apache::thrift::TException"; ++ extends = " : public apache::thrift::TException"; + } else { +- if (is_user_struct && !gen_templates_) { +- extends = " : public virtual ::apache::thrift::TBase"; ++ if (!gen_templates_) { ++ extends = " : public virtual apache::thrift::TBase"; + } + } + +@@ -1130,9 +1132,15 @@ void t_cpp_generator::generate_struct_definition(ofstream& out, + vector::const_iterator m_iter; + const vector& members = tstruct->get_members(); + ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } ++ + // Destructor + if (tstruct->annotations_.find("final") == tstruct->annotations_.end()) { +- force_cpp_out << endl << indent() << tstruct->get_name() << "::~" << tstruct->get_name() ++ force_cpp_out << endl << indent() << method_prefix << ++ tstruct->get_name() << "::~" << tstruct->get_name() + << "() throw() {" << endl; + indent_up(); + +@@ -1145,12 +1153,14 @@ void t_cpp_generator::generate_struct_definition(ofstream& out, + for (m_iter = members.begin(); m_iter != members.end(); ++m_iter) { + if (is_reference((*m_iter))) { + std::string type = type_name((*m_iter)->get_type()); +- out << endl << indent() << "void " << tstruct->get_name() << "::__set_" ++ out << endl << indent() << "void " << method_prefix ++ << tstruct->get_name() << "::__set_" + << (*m_iter)->get_name() << "(boost::shared_ptr<" + << type_name((*m_iter)->get_type(), false, false) << ">"; + out << " val) {" << endl; + } else { +- out << endl << indent() << "void " << tstruct->get_name() << "::__set_" ++ out << endl << indent() << "void " << method_prefix ++ << tstruct->get_name() << "::__set_" + << (*m_iter)->get_name() << "(" << type_name((*m_iter)->get_type(), false, true); + out << " val) {" << endl; + } +@@ -1177,11 +1187,16 @@ void t_cpp_generator::generate_struct_definition(ofstream& out, + * @param tstruct The struct + */ + void t_cpp_generator::generate_struct_reader(ofstream& out, t_struct* tstruct, bool pointers) { ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } ++ + if (gen_templates_) { + out << indent() << "template " << endl << indent() << "uint32_t " +- << tstruct->get_name() << "::read(Protocol_* iprot) {" << endl; ++ << method_prefix << tstruct->get_name() << "::read(Protocol_* iprot) {" << endl; + } else { +- indent(out) << "uint32_t " << tstruct->get_name() ++ indent(out) << "uint32_t " << method_prefix << tstruct->get_name() + << "::read(::apache::thrift::protocol::TProtocol* iprot) {" << endl; + } + indent_up(); +@@ -1301,14 +1316,18 @@ void t_cpp_generator::generate_struct_reader(ofstream& out, t_struct* tstruct, b + */ + void t_cpp_generator::generate_struct_writer(ofstream& out, t_struct* tstruct, bool pointers) { + string name = tstruct->get_name(); ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + if (gen_templates_) { + out << indent() << "template " << endl << indent() << "uint32_t " +- << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; ++ << method_prefix << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; + } else { +- indent(out) << "uint32_t " << tstruct->get_name() ++ indent(out) << "uint32_t " << method_prefix << tstruct->get_name() + << "::write(::apache::thrift::protocol::TProtocol* oprot) const {" << endl; + } + indent_up(); +@@ -1369,14 +1388,18 @@ void t_cpp_generator::generate_struct_result_writer(ofstream& out, + t_struct* tstruct, + bool pointers) { + string name = tstruct->get_name(); ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } + const vector& fields = tstruct->get_sorted_members(); + vector::const_iterator f_iter; + + if (gen_templates_) { + out << indent() << "template " << endl << indent() << "uint32_t " +- << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; ++ << method_prefix << tstruct->get_name() << "::write(Protocol_* oprot) const {" << endl; + } else { +- indent(out) << "uint32_t " << tstruct->get_name() ++ indent(out) << "uint32_t " << method_prefix << tstruct->get_name() + << "::write(::apache::thrift::protocol::TProtocol* oprot) const {" << endl; + } + indent_up(); +@@ -1385,18 +1408,7 @@ void t_cpp_generator::generate_struct_result_writer(ofstream& out, + + indent(out) << "xfer += oprot->writeStructBegin(\"" << name << "\");" << endl; + +- bool first = true; + for (f_iter = fields.begin(); f_iter != fields.end(); ++f_iter) { +- if (first) { +- first = false; +- out << endl << indent() << "if "; +- } else { +- out << " else if "; +- } +- +- out << "(this->__isset." << (*f_iter)->get_name() << ") {" << endl; +- +- indent_up(); + + // Write field header + out << indent() << "xfer += oprot->writeFieldBegin(" +@@ -1410,9 +1422,6 @@ void t_cpp_generator::generate_struct_result_writer(ofstream& out, + } + // Write field closer + indent(out) << "xfer += oprot->writeFieldEnd();" << endl; +- +- indent_down(); +- indent(out) << "}"; + } + + // Write the struct map +@@ -1478,9 +1487,13 @@ void t_cpp_generator::generate_struct_ostream_operator(std::ofstream& out, t_str + } + + void t_cpp_generator::generate_struct_print_method_decl(std::ofstream& out, t_struct* tstruct) { ++ string method_prefix = ""; ++ if (service_name_ != "") { ++ method_prefix = service_name_ + "::"; ++ } + out << "void "; + if (tstruct) { +- out << tstruct->get_name() << "::"; ++ out << method_prefix << tstruct->get_name() << "::"; + } + out << "printTo(std::ostream& out) const"; + } +@@ -1601,11 +1614,13 @@ void t_cpp_generator::generate_exception_what_method(std::ofstream& out, t_struc + */ + void t_cpp_generator::generate_service(t_service* tservice) { + string svcname = tservice->get_name(); ++ string ns = tservice->get_program()->get_namespace("cpp"); + + // Make output files +- string f_header_name = get_out_dir() + svcname + ".h"; ++ string f_header_name = get_out_dir() + svcname + ".grpc.thrift.h"; + f_header_.open(f_header_name.c_str()); + ++ + // Print header file includes + f_header_ << autogen_comment(); + f_header_ << "#ifndef " << svcname << "_H" << endl << "#define " << svcname << "_H" << endl +@@ -1621,15 +1636,38 @@ void t_cpp_generator::generate_service(t_service* tservice) { + f_header_ << "#include " << endl; + } + f_header_ << "#include " << endl; ++ + f_header_ << "#include \"" << get_include_prefix(*get_program()) << program_name_ << "_types.h\"" + << endl; + + t_service* extends_service = tservice->get_extends(); +- if (extends_service != NULL) { ++ if (extends_service != nullptr) { + f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) +- << extends_service->get_name() << ".h\"" << endl; ++ << extends_service->get_name() << ".grpc.thrift.h\"" << endl; + } + ++ ++ f_header_ << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl; ++ ++ ++ f_header_ << ++ endl << ++ "namespace grpc {" << endl << ++ "class CompletionQueue;" << endl << ++ "class Channel;" << endl << ++ "class RpcService;" << endl << ++ "class ServerCompletionQueue;" << endl << ++ "class ServerContext;" << endl << ++ "}" << endl; ++ + f_header_ << endl << ns_open_ << endl << endl; + + f_header_ << "#ifdef _WIN32\n" +@@ -1638,10 +1676,13 @@ void t_cpp_generator::generate_service(t_service* tservice) { + "#endif\n\n"; + + // Service implementation file includes +- string f_service_name = get_out_dir() + svcname + ".cpp"; ++ string f_service_name = get_out_dir() + svcname + ".grpc.thrift.cpp"; + f_service_.open(f_service_name.c_str()); + f_service_ << autogen_comment(); +- f_service_ << "#include \"" << get_include_prefix(*get_program()) << svcname << ".h\"" << endl; ++ ++ f_service_ << "#include \"" << ++ get_include_prefix(*get_program()) << svcname << ".grpc.thrift.h\"" << endl; ++ + if (gen_cob_style_) { + f_service_ << "#include \"thrift/async/TAsyncChannel.h\"" << endl; + } +@@ -1652,7 +1693,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { + string f_service_tcc_name = get_out_dir() + svcname + ".tcc"; + f_service_tcc_.open(f_service_tcc_name.c_str()); + f_service_tcc_ << autogen_comment(); +- f_service_tcc_ << "#include \"" << get_include_prefix(*get_program()) << svcname << ".h\"" ++ f_service_tcc_ << "#include \"" << get_include_prefix(*get_program()) << svcname << ".grpc.thrift.h\"" + << endl; + + f_service_tcc_ << "#ifndef " << svcname << "_TCC" << endl << "#define " << svcname << "_TCC" +@@ -1663,19 +1704,66 @@ void t_cpp_generator::generate_service(t_service* tservice) { + } + } + ++ f_service_ << ++ endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ "#include " << endl << ++ endl; ++ + f_service_ << endl << ns_open_ << endl << endl; + f_service_tcc_ << endl << ns_open_ << endl << endl; + ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ ++ f_service_ << ++ "static const char* " << service_name_ << "_method_names[] = {" << endl; ++ ++ ++ indent_up(); ++ ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; ++ } ++ ++ if (extends_service != nullptr) { ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ ++ for ( f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "\"/" << extends_service->get_program()->get_namespace("cpp") << ++ "." << extends_service->get_name() << "/" << (*f_iter)->get_name() << "\"," << endl; ++ } ++ } ++ ++ indent_down(); ++ f_service_ << ++ "};" << endl; ++ ++ // Generate service class ++ if ( extends_service != nullptr ) { ++ f_header_ << "class " << service_name_ << " : public " << ++ extends_service->get_name() << " {" << endl << ++ "public:" << endl; ++ } ++ else { ++ f_header_ << "class " << service_name_ << "{" << endl << ++ "public:" << endl; ++ } ++ + // Generate all the components +- generate_service_interface(tservice, ""); +- generate_service_interface_factory(tservice, ""); +- generate_service_null(tservice, ""); + generate_service_helpers(tservice); +- generate_service_client(tservice, ""); +- generate_service_processor(tservice, ""); +- generate_service_multiface(tservice); +- generate_service_skeleton(tservice); +- generate_service_client(tservice, "Concurrent"); ++ generate_service_interface(tservice, ""); ++ generate_service_stub_interface(tservice); ++ generate_service_stub(tservice); + + // Generate all the cob components + if (gen_cob_style_) { +@@ -1688,10 +1776,14 @@ void t_cpp_generator::generate_service(t_service* tservice) { + generate_service_async_skeleton(tservice); + } + ++ // Close service class ++ f_header_ << "};" << endl; ++ + f_header_ << "#ifdef _WIN32\n" + " #pragma warning( pop )\n" + "#endif\n\n"; + ++ + // Close the namespace + f_service_ << ns_close_ << endl << endl; + f_service_tcc_ << ns_close_ << endl << endl; +@@ -1729,15 +1821,11 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { + string name_orig = ts->get_name(); + + // TODO(dreiss): Why is this stuff not in generate_function_helpers? +- ts->set_name(tservice->get_name() + "_" + (*f_iter)->get_name() + "_args"); ++ ts->set_name((*f_iter)->get_name() + "Req"); + generate_struct_declaration(f_header_, ts, false); +- generate_struct_definition(out, f_service_, ts, false); ++ generate_struct_definition(out, f_service_, ts, true); + generate_struct_reader(out, ts); + generate_struct_writer(out, ts); +- ts->set_name(tservice->get_name() + "_" + (*f_iter)->get_name() + "_pargs"); +- generate_struct_declaration(f_header_, ts, false, true, false, true); +- generate_struct_definition(out, f_service_, ts, false); +- generate_struct_writer(out, ts, true); + ts->set_name(name_orig); + + generate_function_helpers(tservice, *f_iter); +@@ -1745,13 +1833,210 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { + } + + /** ++ * Generates a service Stub Interface ++ * ++ * @param tservice The service to generate a stub for. ++ * ++ */ ++void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { ++ ++ string extends = ""; ++ if (tservice->get_extends() != nullptr) { ++ extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; ++ } ++ ++ f_header_ << ++ endl << ++ "class StubInterface " << extends << " {" << endl; ++ indent_up(); ++ f_header_ << ++ " public:" << endl << ++ indent() << "virtual ~StubInterface() {}" << endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "virtual ::grpc::Status " << function_name << ++ "(::grpc::ClientContext* context, const " << function_name << ++ "Req& request, " << function_name << "Resp* response) = 0;" << endl; ++ } ++ indent_down(); ++ f_header_ << ++ "};" << endl << endl; ++ ++} ++void t_cpp_generator::generate_service_stub(t_service* tservice) { ++ f_header_ << ++ endl << ++ "class Stub : public StubInterface {" << ++ endl; ++ ++ indent_up(); ++ f_header_ << ++ " public:" << endl << ++ indent() << "Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel);" << ++ endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "::grpc::Status " << function_name << ++ "(::grpc::ClientContext* context, const " << function_name << ++ "Req& request, " << function_name << "Resp* response) override;" << endl; ++ } ++ ++ t_service* extends_service = tservice->get_extends(); ++ if (extends_service != nullptr) { ++ // generate inherited methods ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "::grpc::Status " << function_name << ++ "(::grpc::ClientContext* context, const " << function_name << ++ "Req& request, " << function_name << "Resp* response) override;" << endl; ++ } ++ } ++ ++ f_header_ << ++ endl << ++ " private:" << endl << ++ indent() << "std::shared_ptr< ::grpc::ChannelInterface> channel_;" << ++ endl; ++ ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_header_ << ++ indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; ++ } ++ ++ if (extends_service != nullptr) { ++ // generate inherited methods ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_header_ << ++ indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; ++ } ++ } ++ ++ indent_down(); ++ f_header_ << ++ "};" << endl << endl; ++ ++ // generate the implementaion of Stub ++ f_service_ << ++ endl << ++ service_name_ << "::Stub::Stub(const std::shared_ptr< ::grpc::ChannelInterface>& channel)" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << ": channel_(channel)" << endl; ++ int i=0; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter , ++i) { ++ f_service_ << ++ indent() << ++ ", rpcmethod_" << (*f_iter)->get_name() << "_(" << ++ service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; ++ } ++ ++ if (extends_service != nullptr) { ++ // generate inherited methods ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { ++ f_service_ << ++ indent() << ++ ", rpcmethod_" << (*f_iter)->get_name() << "_(" << ++ service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; ++ } ++ } ++ f_service_ << ++ indent() << "{}" << endl; ++ indent_down(); ++ ++ // generate NewStub ++ f_header_ << ++ endl << ++ "static std::unique_ptr NewStub(const std::shared_ptr\ ++ < ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options = ::grpc::StubOptions());" << ++ endl; ++ ++ // generate NewStub Implementation ++ f_service_ << ++ endl << ++ "std::unique_ptr< " << service_name_ << "::Stub> " << service_name_ << "::NewStub(const std::shared_ptr\ ++ < ::grpc::ChannelInterface>& channel, const ::grpc::StubOptions& options) {" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "std::unique_ptr< " << service_name_ << "::Stub> stub(new " << service_name_ << ++ "::Stub(channel));" << endl << ++ indent() << "return stub;" << endl; ++ indent_down(); ++ f_service_ << ++ "}" << endl; ++ ++ // generate stub methods ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_service_ << ++ endl << ++ "::grpc::Status " << service_name_ << "::Stub::" << function_name << ++ "(::grpc::ClientContext* context, const " << service_name_ << "::" << ++ function_name << "Req& request, " << service_name_ << "::" << ++ function_name << "Resp* response) {" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_" << ++ function_name << "_, context, request, response);" << endl; ++ indent_down(); ++ ++ f_service_ << ++ "}" << endl; ++ ++ } ++ ++ if (extends_service != nullptr) { ++ vector functions = extends_service->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_service_ << ++ endl << ++ "::grpc::Status " << service_name_ << "::Stub::" << function_name << ++ "(::grpc::ClientContext* context, const " << service_name_ << "::" << ++ function_name << "Req& request, " << service_name_ << "::" << ++ function_name << "Resp* response) {" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "return ::grpc::BlockingUnaryCall(channel_.get(), rpcmethod_" << ++ function_name << "_, context, request, response);" << endl; ++ indent_down(); ++ ++ f_service_ << ++ "}" << endl; ++ ++ } ++ } ++ ++} ++ ++ ++/** + * Generates a service interface definition. + * + * @param tservice The service to generate a header definition for + */ + void t_cpp_generator::generate_service_interface(t_service* tservice, string style) { + +- string service_if_name = service_name_ + style + "If"; ++ string service_if_name = "Service"; + if (style == "CobCl") { + // Forward declare the client. + string client_name = service_name_ + "CobClient"; +@@ -1765,12 +2050,14 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + + string extends = ""; + if (tservice->get_extends() != NULL) { +- extends = " : virtual public " + type_name(tservice->get_extends()) + style + "If"; ++ extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; + if (style == "CobCl" && gen_templates_) { + // TODO(simpkins): If gen_templates_ is enabled, we currently assume all + // parent services were also generated with templates enabled. + extends += "T"; + } ++ } else { ++ extends = " : public ::grpc::Service"; + } + + if (style == "CobCl" && gen_templates_) { +@@ -1778,7 +2065,9 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + } + f_header_ << "class " << service_if_name << extends << " {" << endl << " public:" << endl; + indent_up(); +- f_header_ << indent() << "virtual ~" << service_if_name << "() {}" << endl; ++ ++ f_header_ << indent() << "Service();" << endl; ++ f_header_ << indent() << "virtual ~Service();" << endl; + + vector functions = tservice->get_functions(); + vector::iterator f_iter; +@@ -1786,7 +2075,12 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + if ((*f_iter)->has_doc()) + f_header_ << endl; + generate_java_doc(f_header_, *f_iter); +- f_header_ << indent() << "virtual " << function_signature(*f_iter, style) << " = 0;" << endl; ++ ++ string function_name = (*f_iter)->get_name(); ++ f_header_ << ++ indent() << "virtual ::grpc::Status " << function_name << ++ "(::grpc::ServerContext* context, const "<< function_name << ++ "Req* request, "<< function_name << "Resp* response);" << endl; + } + indent_down(); + f_header_ << "};" << endl << endl; +@@ -1797,6 +2091,66 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + f_header_ << "typedef " << service_if_name << "< ::apache::thrift::protocol::TProtocol> " + << service_name_ << style << "If;" << endl << endl; + } ++ ++ // generate the service interface implementations ++ ++ f_service_ << ++ endl << ++ service_name_ << "::Service::Service() {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "(void)" << service_name_ << "_method_names;" << endl; ++ uint32_t i=0; ++ for(i=0;iget_name(); ++ f_service_ << ++ endl << ++ indent() << "AddMethod(new ::grpc::RpcServiceMethod(" << endl; ++ indent_up(); ++ ++ f_service_ << ++ indent() << service_name_ << "_method_names[" << i << "]," << endl << ++ indent() << "::grpc::RpcMethod::NORMAL_RPC," << endl << ++ indent() << "new ::grpc::RpcMethodHandler< " << service_name_ << "::Service, " << ++ service_name_ << "::" << function_name << "Req, " << service_name_ << "::" << ++ function_name << "Resp>(" << endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << "std::mem_fn(&" << service_name_ << "::Service::" << function_name << "), this)));" << endl; ++ ++ indent_down(); ++ indent_down(); ++ } ++ ++ indent_down(); ++ f_service_ << ++ "}" << endl; ++ ++ f_service_ << ++ endl << ++ service_name_ << "::Service::~Service() {" << endl << ++ "}" << endl; ++ ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ string function_name = (*f_iter)->get_name(); ++ f_service_ << ++ endl << ++ "::grpc::Status " << service_name_ << "::Service::" << function_name << ++ "(::grpc::ServerContext* context, const " << service_name_ << "::" << function_name << ++ "Req* request, " << service_name_ << "::" << function_name << "Resp* response) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "(void) context;" << endl << ++ indent() << "(void) request;" << endl << ++ indent() << "(void) response;" << endl << ++ indent() << "return ::grpc::Status(::grpc::StatusCode::UNIMPLEMENTED,\"\");" << endl; ++ indent_down(); ++ ++ f_service_ << ++ "}" << endl; ++ } ++ + } + + /** +@@ -3095,7 +3449,7 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* + + std::ofstream& out = (gen_templates_ ? f_service_tcc_ : f_service_); + +- t_struct result(program_, tservice->get_name() + "_" + tfunction->get_name() + "_result"); ++ t_struct result(program_, tfunction->get_name() + "Resp"); + t_field success(tfunction->get_returntype(), "success", 0); + if (!tfunction->get_returntype()->is_void()) { + result.append(&success); +@@ -3109,17 +3463,9 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* + } + + generate_struct_declaration(f_header_, &result, false); +- generate_struct_definition(out, f_service_, &result, false); ++ generate_struct_definition(out, f_service_, &result, true); + generate_struct_reader(out, &result); + generate_struct_result_writer(out, &result); +- +- result.set_name(tservice->get_name() + "_" + tfunction->get_name() + "_presult"); +- generate_struct_declaration(f_header_, &result, false, true, true, gen_cob_style_); +- generate_struct_definition(out, f_service_, &result, false); +- generate_struct_reader(out, &result, true); +- if (gen_cob_style_) { +- generate_struct_writer(out, &result, true); +- } + } + + /** +@@ -3162,8 +3508,8 @@ void t_cpp_generator::generate_process_function(t_service* tservice, + << endl; + scope_up(out); + +- string argsname = tservice->get_name() + "_" + tfunction->get_name() + "_args"; +- string resultname = tservice->get_name() + "_" + tfunction->get_name() + "_result"; ++ string argsname = tfunction->get_name() + "Req"; ++ string resultname = tfunction->get_name() + "Resp"; + + if (tfunction->is_oneway() && !unnamed_oprot_seqid) { + out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; +@@ -3320,7 +3666,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, + out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; + } + +- out << indent() << tservice->get_name() + "_" + tfunction->get_name() << "_args args;" << endl ++ out << indent() << tfunction->get_name() << "Req args;" << endl + << indent() << "void* ctx = NULL;" << endl << indent() + << "if (this->eventHandler_.get() != NULL) {" << endl << indent() + << " ctx = this->eventHandler_->getContext(" << service_func_name << ", NULL);" << endl +@@ -3487,7 +3833,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, + << "this->eventHandler_.get(), ctx, " << service_func_name << ");" << endl << endl; + + // Throw the TDelayedException, and catch the result +- out << indent() << tservice->get_name() << "_" << tfunction->get_name() << "_result result;" ++ out << indent() << tfunction->get_name() << "Resp result;" + << endl << endl << indent() << "try {" << endl; + indent_up(); + out << indent() << "_throw->throw_it();" << endl << indent() << "return cob(false);" +diff --git a/tutorial/cpp/CMakeLists.txt b/tutorial/cpp/CMakeLists.txt +deleted file mode 100644 +index 8a3d085..0000000 +--- a/tutorial/cpp/CMakeLists.txt ++++ /dev/null +@@ -1,53 +0,0 @@ +-# +-# Licensed to the Apache Software Foundation (ASF) under one +-# or more contributor license agreements. See the NOTICE file +-# distributed with this work for additional information +-# regarding copyright ownership. The ASF licenses this file +-# to you under the Apache License, Version 2.0 (the +-# "License"); you may not use this file except in compliance +-# with the License. You may obtain a copy of the License at +-# +-# http://www.apache.org/licenses/LICENSE-2.0 +-# +-# Unless required by applicable law or agreed to in writing, +-# software distributed under the License is distributed on an +-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-# KIND, either express or implied. See the License for the +-# specific language governing permissions and limitations +-# under the License. +-# +- +-find_package(Boost 1.53.0 REQUIRED) +-include_directories(SYSTEM "${Boost_INCLUDE_DIRS}") +- +-#Make sure gen-cpp files can be included +-include_directories("${CMAKE_CURRENT_BINARY_DIR}") +-include_directories("${CMAKE_CURRENT_BINARY_DIR}/gen-cpp") +-include_directories("${PROJECT_SOURCE_DIR}/lib/cpp/src") +- +-include(ThriftMacros) +- +-set(tutorialgencpp_SOURCES +- gen-cpp/Calculator.cpp +- gen-cpp/SharedService.cpp +- gen-cpp/shared_constants.cpp +- gen-cpp/shared_types.cpp +- gen-cpp/tutorial_constants.cpp +- gen-cpp/tutorial_types.cpp +-) +-add_library(tutorialgencpp STATIC ${tutorialgencpp_SOURCES}) +-LINK_AGAINST_THRIFT_LIBRARY(tutorialgencpp thrift) +- +-add_custom_command(OUTPUT gen-cpp/Calculator.cpp gen-cpp/SharedService.cpp gen-cpp/shared_constants.cpp gen-cpp/shared_types.cpp gen-cpp/tutorial_constants.cpp gen-cpp/tutorial_types.cpp +- COMMAND ${THRIFT_COMPILER} --gen cpp -r ${PROJECT_SOURCE_DIR}/tutorial/tutorial.thrift +-) +- +-add_executable(TutorialServer CppServer.cpp) +-target_link_libraries(TutorialServer tutorialgencpp) +-LINK_AGAINST_THRIFT_LIBRARY(TutorialServer thrift) +-target_link_libraries(TutorialServer ${ZLIB_LIBRARIES}) +- +-add_executable(TutorialClient CppClient.cpp) +-target_link_libraries(TutorialClient tutorialgencpp) +-LINK_AGAINST_THRIFT_LIBRARY(TutorialClient thrift) +-target_link_libraries(TutorialClient ${ZLIB_LIBRARIES}) +diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/CppClient.cpp +index 2763fee..c41604e 100644 +--- a/tutorial/cpp/CppClient.cpp ++++ b/tutorial/cpp/CppClient.cpp +@@ -1,80 +1,94 @@ + /* +- * Licensed to the Apache Software Foundation (ASF) under one +- * or more contributor license agreements. See the NOTICE file +- * distributed with this work for additional information +- * regarding copyright ownership. The ASF licenses this file +- * to you under the Apache License, Version 2.0 (the +- * "License"); you may not use this file except in compliance +- * with the License. You may obtain a copy of the License at + * +- * http://www.apache.org/licenses/LICENSE-2.0 ++ * Copyright 2015, Google Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above ++ * copyright notice, this list of conditions and the following disclaimer ++ * in the documentation and/or other materials provided with the ++ * distribution. ++ * * Neither the name of Google Inc. nor the names of its ++ * contributors may be used to endorse or promote products derived from ++ * this software without specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * +- * Unless required by applicable law or agreed to in writing, +- * software distributed under the License is distributed on an +- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +- * KIND, either express or implied. See the License for the +- * specific language governing permissions and limitations +- * under the License. + */ + + #include ++#include ++#include + +-#include +-#include +-#include +- +-#include "../gen-cpp/Calculator.h" ++#include + +-using namespace std; +-using namespace apache::thrift; +-using namespace apache::thrift::protocol; +-using namespace apache::thrift::transport; ++#include "gen-cpp/Greeter.grpc.thrift.h" + +-using namespace tutorial; +-using namespace shared; ++using grpc::Channel; ++using grpc::ClientContext; ++using grpc::Status; ++using test::Greeter; ++using namespace test; + +-int main() { +- boost::shared_ptr socket(new TSocket("localhost", 9090)); +- boost::shared_ptr transport(new TBufferedTransport(socket)); +- boost::shared_ptr protocol(new TBinaryProtocol(transport)); +- CalculatorClient client(protocol); ++class GreeterClient { ++ public: ++ GreeterClient(std::shared_ptr channel) ++ : stub_(Greeter::NewStub(channel)) {} + +- try { +- transport->open(); ++ // Assambles the client's payload, sends it and presents the response back ++ // from the server. ++ std::string SayHello(const std::string& user) { ++ // Data we are sending to the server. ++ Greeter::SayHelloReq req; ++ req.request.name = user; + +- client.ping(); +- cout << "ping()" << endl; ++ // Container for the data we expect from the server. ++ Greeter::SayHelloResp reply; + +- cout << "1 + 1 = " << client.add(1, 1) << endl; ++ // Context for the client. It could be used to convey extra information to ++ // the server and/or tweak certain RPC behaviors. ++ ClientContext context; + +- Work work; +- work.op = Operation::DIVIDE; +- work.num1 = 1; +- work.num2 = 0; ++ // The actual RPC. ++ Status status = stub_->SayHello(&context, req, &reply); + +- try { +- client.calculate(1, work); +- cout << "Whoa? We can divide by zero!" << endl; +- } catch (InvalidOperation& io) { +- cout << "InvalidOperation: " << io.why << endl; +- // or using generated operator<<: cout << io << endl; +- // or by using std::exception native method what(): cout << io.what() << endl; ++ // Act upon its status. ++ if (status.ok()) { ++ return reply.success.message; ++ } else { ++ return "RPC failed"; + } ++ } + +- work.op = Operation::SUBTRACT; +- work.num1 = 15; +- work.num2 = 10; +- int32_t diff = client.calculate(1, work); +- cout << "15 - 10 = " << diff << endl; ++ private: ++ std::unique_ptr stub_; ++}; + +- // Note that C++ uses return by reference for complex types to avoid +- // costly copy construction +- SharedStruct ss; +- client.getStruct(ss, 1); +- cout << "Received log: " << ss << endl; ++int main() { ++ // Instantiate the client. It requires a channel, out of which the actual RPCs ++ // are created. This channel models a connection to an endpoint (in this case, ++ // localhost at port 50051). We indicate that the channel isn't authenticated ++ // (use of InsecureChannelCredentials()). ++ GreeterClient greeter(grpc::CreateChannel( ++ "localhost:50051", grpc::InsecureChannelCredentials())); ++ std::string user("world"); ++ std::string reply = greeter.SayHello(user); ++ std::cout << "Greeter received: " << reply << std::endl; + +- transport->close(); +- } catch (TException& tx) { +- cout << "ERROR: " << tx.what() << endl; +- } ++ return 0; + } +diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/CppServer.cpp +index eafffa9..c838b61 100644 +--- a/tutorial/cpp/CppServer.cpp ++++ b/tutorial/cpp/CppServer.cpp +@@ -1,181 +1,87 @@ + /* +- * Licensed to the Apache Software Foundation (ASF) under one +- * or more contributor license agreements. See the NOTICE file +- * distributed with this work for additional information +- * regarding copyright ownership. The ASF licenses this file +- * to you under the Apache License, Version 2.0 (the +- * "License"); you may not use this file except in compliance +- * with the License. You may obtain a copy of the License at + * +- * http://www.apache.org/licenses/LICENSE-2.0 ++ * Copyright 2015, Google Inc. ++ * All rights reserved. ++ * ++ * Redistribution and use in source and binary forms, with or without ++ * modification, are permitted provided that the following conditions are ++ * met: ++ * ++ * * Redistributions of source code must retain the above copyright ++ * notice, this list of conditions and the following disclaimer. ++ * * Redistributions in binary form must reproduce the above ++ * copyright notice, this list of conditions and the following disclaimer ++ * in the documentation and/or other materials provided with the ++ * distribution. ++ * * Neither the name of Google Inc. nor the names of its ++ * contributors may be used to endorse or promote products derived from ++ * this software without specific prior written permission. ++ * ++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ++ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ++ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ++ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ++ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ++ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ++ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ++ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ++ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * +- * Unless required by applicable law or agreed to in writing, +- * software distributed under the License is distributed on an +- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +- * KIND, either express or implied. See the License for the +- * specific language governing permissions and limitations +- * under the License. + */ + +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- + #include +-#include +-#include +- +-#include "../gen-cpp/Calculator.h" ++#include ++#include + +-using namespace std; +-using namespace apache::thrift; +-using namespace apache::thrift::concurrency; +-using namespace apache::thrift::protocol; +-using namespace apache::thrift::transport; +-using namespace apache::thrift::server; ++#include + +-using namespace tutorial; +-using namespace shared; ++#include "gen-cpp/Greeter.grpc.thrift.h" ++#include + +-class CalculatorHandler : public CalculatorIf { +-public: +- CalculatorHandler() {} ++using grpc::Server; ++using grpc::ServerBuilder; ++using grpc::ServerContext; ++using grpc::Status; ++using test::Greeter; + +- void ping() { cout << "ping()" << endl; } ++using namespace grpc; ++using namespace test; + +- int32_t add(const int32_t n1, const int32_t n2) { +- cout << "add(" << n1 << ", " << n2 << ")" << endl; +- return n1 + n2; +- } ++// Logic and data behind the server's behavior. ++class GreeterServiceImpl final : public Greeter::Service { ++ Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, ++ Greeter::SayHelloResp* reply) override { ++ std::string prefix("Hello "); + +- int32_t calculate(const int32_t logid, const Work& work) { +- cout << "calculate(" << logid << ", " << work << ")" << endl; +- int32_t val; ++ reply->success.message = prefix + request->request.name; + +- switch (work.op) { +- case Operation::ADD: +- val = work.num1 + work.num2; +- break; +- case Operation::SUBTRACT: +- val = work.num1 - work.num2; +- break; +- case Operation::MULTIPLY: +- val = work.num1 * work.num2; +- break; +- case Operation::DIVIDE: +- if (work.num2 == 0) { +- InvalidOperation io; +- io.whatOp = work.op; +- io.why = "Cannot divide by 0"; +- throw io; +- } +- val = work.num1 / work.num2; +- break; +- default: +- InvalidOperation io; +- io.whatOp = work.op; +- io.why = "Invalid Operation"; +- throw io; +- } +- +- SharedStruct ss; +- ss.key = logid; +- ss.value = to_string(val); +- +- log[logid] = ss; +- +- return val; ++ return Status::OK; + } +- +- void getStruct(SharedStruct& ret, const int32_t logid) { +- cout << "getStruct(" << logid << ")" << endl; +- ret = log[logid]; +- } +- +- void zip() { cout << "zip()" << endl; } +- +-protected: +- map log; + }; + +-/* +- CalculatorIfFactory is code generated. +- CalculatorCloneFactory is useful for getting access to the server side of the +- transport. It is also useful for making per-connection state. Without this +- CloneFactory, all connections will end up sharing the same handler instance. +-*/ +-class CalculatorCloneFactory : virtual public CalculatorIfFactory { +- public: +- virtual ~CalculatorCloneFactory() {} +- virtual CalculatorIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) +- { +- boost::shared_ptr sock = boost::dynamic_pointer_cast(connInfo.transport); +- cout << "Incoming connection\n"; +- cout << "\tSocketInfo: " << sock->getSocketInfo() << "\n"; +- cout << "\tPeerHost: " << sock->getPeerHost() << "\n"; +- cout << "\tPeerAddress: " << sock->getPeerAddress() << "\n"; +- cout << "\tPeerPort: " << sock->getPeerPort() << "\n"; +- return new CalculatorHandler; +- } +- virtual void releaseHandler( ::shared::SharedServiceIf* handler) { +- delete handler; +- } +-}; ++void RunServer() { ++ std::string server_address("0.0.0.0:50051"); ++ GreeterServiceImpl service; ++ ++ ServerBuilder builder; ++ // Listen on the given address without any authentication mechanism. ++ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); ++ // Register "service" as the instance through which we'll communicate with ++ // clients. In this case it corresponds to an *synchronous* service. ++ builder.RegisterService(&service); ++ // Finally assemble the server. ++ std::unique_ptr server(builder.BuildAndStart()); ++ std::cout << "Server listening on " << server_address << std::endl; ++ ++ // Wait for the server to shutdown. Note that some other thread must be ++ // responsible for shutting down the server for this call to ever return. ++ server->Wait(); ++} + + int main() { +- TThreadedServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), //port +- boost::make_shared(), +- boost::make_shared()); +- +- /* +- // if you don't need per-connection state, do the following instead +- TThreadedServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), //port +- boost::make_shared(), +- boost::make_shared()); +- */ +- +- /** +- * Here are some alternate server types... +- +- // This server only allows one connection at a time, but spawns no threads +- TSimpleServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), +- boost::make_shared(), +- boost::make_shared()); +- +- const int workerCount = 4; +- +- boost::shared_ptr threadManager = +- ThreadManager::newSimpleThreadManager(workerCount); +- threadManager->threadFactory( +- boost::make_shared()); +- threadManager->start(); +- +- // This server allows "workerCount" connection at a time, and reuses threads +- TThreadPoolServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), +- boost::make_shared(), +- boost::make_shared(), +- threadManager); +- */ ++ RunServer(); + +- cout << "Starting the server..." << endl; +- server.serve(); +- cout << "Done." << endl; + return 0; + } +diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am +index 184a69d..581f75e 100755 +--- a/tutorial/cpp/Makefile.am ++++ b/tutorial/cpp/Makefile.am +@@ -18,44 +18,38 @@ + # + AUTOMAKE_OPTIONS = subdir-objects serial-tests + +-BUILT_SOURCES = gen-cpp/shared_types.cpp \ +- gen-cpp/tutorial_types.cpp ++BUILT_SOURCES = gen-cpp/test_types.cpp + +-noinst_LTLIBRARIES = libtutorialgencpp.la +-nodist_libtutorialgencpp_la_SOURCES = \ +- gen-cpp/Calculator.cpp \ +- gen-cpp/Calculator.h \ +- gen-cpp/SharedService.cpp \ +- gen-cpp/SharedService.h \ +- gen-cpp/shared_constants.cpp \ +- gen-cpp/shared_constants.h \ +- gen-cpp/shared_types.cpp \ +- gen-cpp/shared_types.h \ +- gen-cpp/tutorial_constants.cpp \ +- gen-cpp/tutorial_constants.h \ +- gen-cpp/tutorial_types.cpp \ +- gen-cpp/tutorial_types.h ++#noinst_LTLIBRARIES = libtutorialgencpp.la ++noinst_LTLIBRARIES = libtestgencpp.la ++nodist_libtestgencpp_la_SOURCES = \ ++ gen-cpp/Greeter.grpc.thrift.cpp \ ++ gen-cpp/Greeter.grpc.thrift.h \ ++ gen-cpp/test_constants.cpp \ ++ gen-cpp/test_constants.h \ ++ gen-cpp/test_types.cpp \ ++ gen-cpp/test_types.h + + + +-libtutorialgencpp_la_LIBADD = $(top_builddir)/lib/cpp/libthrift.la ++libtestgencpp_la_LIBADD = $(top_builddir)/lib/cpp/libthrift.la + + noinst_PROGRAMS = \ +- TutorialServer \ +- TutorialClient ++ TestServer \ ++ TestClient + +-TutorialServer_SOURCES = \ ++TestServer_SOURCES = \ + CppServer.cpp + +-TutorialServer_LDADD = \ +- libtutorialgencpp.la \ ++TestServer_LDADD = \ ++ libtestgencpp.la \ + $(top_builddir)/lib/cpp/libthrift.la + +-TutorialClient_SOURCES = \ ++TestClient_SOURCES = \ + CppClient.cpp + +-TutorialClient_LDADD = \ +- libtutorialgencpp.la \ ++TestClient_LDADD = \ ++ libtestgencpp.la \ + $(top_builddir)/lib/cpp/libthrift.la + + # +@@ -63,21 +57,21 @@ TutorialClient_LDADD = \ + # + THRIFT = $(top_builddir)/compiler/cpp/thrift + +-gen-cpp/Calculator.cpp gen-cpp/SharedService.cpp gen-cpp/shared_constants.cpp gen-cpp/shared_types.cpp gen-cpp/tutorial_constants.cpp gen-cpp/tutorial_types.cpp: $(top_srcdir)/tutorial/tutorial.thrift ++gen-cpp/Greeter.grpc.thrift.cpp gen-cpp/test_constants.cpp gen-cpp/test_types.cpp: $(top_srcdir)/tutorial/cpp/test.thrift + $(THRIFT) --gen cpp -r $< + + AM_CPPFLAGS = $(BOOST_CPPFLAGS) $(LIBEVENT_CPPFLAGS) -I$(top_srcdir)/lib/cpp/src -Igen-cpp + AM_CXXFLAGS = -Wall -Wextra -pedantic +-AM_LDFLAGS = $(BOOST_LDFLAGS) $(LIBEVENT_LDFLAGS) ++AM_LDFLAGS = $(BOOST_LDFLAGS) $(LIBEVENT_LDFLAGS) `pkg-config --libs grpc++ grpc` -lpthread -ldl -lgrpc + + clean-local: +- $(RM) gen-cpp/* ++ $(RM) -r gen-cpp + +-tutorialserver: all +- ./TutorialServer ++testserver: all ++ ./TestServer + +-tutorialclient: all +- ./TutorialClient ++testclient: all ++ ./TestClient + + style-local: + $(CPPSTYLE_CMD) +diff --git a/tutorial/cpp/test.thrift b/tutorial/cpp/test.thrift +new file mode 100644 +index 0000000..de3c9a4 +--- /dev/null ++++ b/tutorial/cpp/test.thrift +@@ -0,0 +1,13 @@ ++namespace cpp test ++ ++struct HelloRequest { ++ 1:string name ++} ++ ++struct HelloResponse { ++ 1:string message ++} ++ ++service Greeter { ++ HelloResponse SayHello(1:HelloRequest request); ++} +\ No newline at end of file +-- +2.8.0.rc3.226.g39d4020 + + +From 1d47ed062e62d136c3db9f6fc1dde9e2f794cf22 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Sun, 31 Jul 2016 16:23:53 -0700 +Subject: [PATCH 3/3] grpc java plugins generator + +for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thrift +--- + compiler/cpp/src/generate/t_java_generator.cc | 906 +++++++++++++++++++++++++- + tutorial/Makefile.am | 8 +- + 2 files changed, 887 insertions(+), 27 deletions(-) + +diff --git a/compiler/cpp/src/generate/t_java_generator.cc b/compiler/cpp/src/generate/t_java_generator.cc +index 2db8cb8..95e1ca8 100644 +--- a/compiler/cpp/src/generate/t_java_generator.cc ++++ b/compiler/cpp/src/generate/t_java_generator.cc +@@ -97,10 +97,10 @@ public: + } else if(iter->second.compare("suppress") == 0) { + suppress_generated_annotations_ = true; + } else { +- throw "unknown option java:" + iter->first + "=" + iter->second; ++ throw "unknown option java:" + iter->first + "=" + iter->second; + } + } else { +- throw "unknown option java:" + iter->first; ++ throw "unknown option java:" + iter->first; + } + } + +@@ -195,6 +195,17 @@ public: + void generate_service_async_server(t_service* tservice); + void generate_process_function(t_service* tservice, t_function* tfunction); + void generate_process_async_function(t_service* tservice, t_function* tfunction); ++ void generate_service_impl_base(t_service* tservice); ++ void generate_method_descriptors(t_service* tservice); ++ void generate_stub(t_service* tservice); ++ void generate_blocking_stub(t_service* tservice); ++ void generate_future_stub(t_service* tservice); ++ void generate_method_ids(t_service* tservice); ++ void generate_method_handlers(t_service* tservice); ++ void generate_service_descriptors(t_service* tservice); ++ void generate_service_builder(t_service* tservice); ++ void generate_arg_ids(t_service* tservice); ++ void generate_message_factory(t_service* tservice); + + void generate_java_union(t_struct* tstruct); + void generate_union_constructor(ofstream& out, t_struct* tstruct); +@@ -307,6 +318,8 @@ public: + std::string java_package(); + std::string java_type_imports(); + std::string java_suppressions(); ++ std::string grpc_imports(); ++ std::string import_extended_service(t_service* tservice); + std::string type_name(t_type* ttype, + bool in_container = false, + bool in_init = false, +@@ -368,7 +381,7 @@ private: + bool use_option_type_; + bool undated_generated_annotations_; + bool suppress_generated_annotations_; +- ++ + }; + + /** +@@ -456,6 +469,35 @@ string t_java_generator::java_suppressions() { + return "@SuppressWarnings({\"cast\", \"rawtypes\", \"serial\", \"unchecked\", \"unused\"})\n"; + } + ++string t_java_generator::grpc_imports() { ++ return ++ string() + ++ "import static io.grpc.stub.ClientCalls.asyncUnaryCall;\n" + ++ "import static io.grpc.stub.ClientCalls.asyncServerStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.asyncClientStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.asyncBidiStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.blockingUnaryCall;\n" + ++ "import static io.grpc.stub.ClientCalls.blockingServerStreamingCall;\n" + ++ "import static io.grpc.stub.ClientCalls.futureUnaryCall;\n" + ++ "import static io.grpc.MethodDescriptor.generateFullMethodName;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncUnaryCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncServerStreamingCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncClientStreamingCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncBidiStreamingCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall;\n" + ++ "import static io.grpc.stub.ServerCalls.asyncUnimplementedStreamingCall;\n" + ++ "import io.grpc.thrift.ThriftUtils;\n\n"; ++} ++ ++string t_java_generator::import_extended_service(t_service* tservice) { ++ if (tservice == nullptr) { ++ return string() + "\n"; ++ } ++ string ns = tservice->get_program()->get_namespace("java"); ++ string extend_service_name = tservice->get_name() + "Grpc"; ++ return string() + "import " + ns + "." + extend_service_name + ";\n\n"; ++} ++ + /** + * Nothing in Java + */ +@@ -2772,25 +2814,51 @@ void t_java_generator::generate_field_value_meta_data(std::ofstream& out, t_type + */ + void t_java_generator::generate_service(t_service* tservice) { + // Make output file +- string f_service_name = package_dir_ + "/" + make_valid_java_filename(service_name_) + ".java"; ++ string f_service_name = package_dir_ + "/" + make_valid_java_filename(service_name_) + "Grpc.java"; + f_service_.open(f_service_name.c_str()); + +- f_service_ << autogen_comment() << java_package() << java_type_imports() << java_suppressions(); ++ f_service_ << ++ autogen_comment() << ++ java_package() << ++ java_type_imports() << ++ grpc_imports() << ++ import_extended_service(tservice->get_extends()); ++ java_suppressions(); ++ ++ f_service_ << ++ "public class " << service_name_ << "Grpc {" << endl << ++ endl; + +- if (!suppress_generated_annotations_) { +- generate_javax_generated_annotation(f_service_); +- } +- f_service_ << "public class " << service_name_ << " {" << endl << endl; + indent_up(); + ++ // generate constructor ++ f_service_ << ++ indent() << "private " << service_name_ << ++ "Grpc() {}" << endl << endl; ++ ++ f_service_ << ++ indent() << "public static final String SERVICE_NAME = " << ++ "\"" << package_name_ << "." << service_name_ << "\";" << endl << endl; ++ + // Generate the three main parts of the service + generate_service_interface(tservice); +- generate_service_async_interface(tservice); +- generate_service_client(tservice); +- generate_service_async_client(tservice); +- generate_service_server(tservice); +- generate_service_async_server(tservice); ++ generate_arg_ids(tservice); ++ generate_message_factory(tservice); ++ generate_service_impl_base(tservice); ++ //generate_service_async_interface(tservice); ++ //generate_service_client(tservice); ++ //generate_service_async_client(tservice); ++ //generate_service_server(tservice); ++ //generate_service_async_server(tservice); + generate_service_helpers(tservice); ++ generate_method_descriptors(tservice); ++ generate_stub(tservice); ++ generate_blocking_stub(tservice); ++ generate_future_stub(tservice); ++ generate_method_ids(tservice); ++ generate_method_handlers(tservice); ++ generate_service_descriptors(tservice); ++ generate_service_builder(tservice); + + indent_down(); + f_service_ << "}" << endl; +@@ -2805,24 +2873,820 @@ void t_java_generator::generate_service(t_service* tservice) { + void t_java_generator::generate_service_interface(t_service* tservice) { + string extends = ""; + string extends_iface = ""; +- if (tservice->get_extends() != NULL) { +- extends = type_name(tservice->get_extends()); +- extends_iface = " extends " + extends + ".Iface"; +- } + + generate_java_doc(f_service_, tservice); +- f_service_ << indent() << "public interface Iface" << extends_iface << " {" << endl << endl; ++ f_service_ << indent() << ++ "@java.lang.Deprecated public static interface " << service_name_; ++ ++ if (tservice->get_extends() != nullptr) { ++ f_service_ << " extends " << tservice->get_extends()->get_name() + "Grpc." << ++ tservice->get_extends()->get_name() << endl; ++ } ++ f_service_ << " {" << endl; ++ ++ indent_up(); ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ //generate_java_doc(f_service_, *f_iter); ++ f_service_ << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << (*f_iter)->get_name() << ++ "_args request," << endl << ++ indent() << " io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << ++ "_result> responseObserver);" << endl << endl; ++ } ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_arg_ids(t_service* tservice) { ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ int i=0; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "private static final int ARG_IN_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ f_service_ << indent() << ++ "private static final int ARG_OUT_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ } ++ f_service_ << endl; ++ ++ if (tservice->get_extends() != nullptr) { ++ f_service_ << indent() << "// ARG IDs for extended service" << endl; ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "private static final int ARG_IN_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ f_service_ << indent() << ++ "private static final int ARG_OUT_METHOD_" << ++ (*f_iter)->get_name() << " = " << ++i << ";" << endl; ++ } ++ f_service_ << endl; ++ } ++} ++ ++void t_java_generator::generate_message_factory(t_service* tservice) { ++ f_service_ << indent() << ++ "private static final class ThriftMessageFactory>" << endl << indent() << ++ " implements io.grpc.thrift.MessageFactory {" << endl; ++ indent_up(); ++ f_service_ << indent() << ++ "private final int id;" << endl << endl; ++ f_service_ << endl; ++ ++ f_service_ << indent() << ++ "ThriftMessageFactory(int id) {" << endl << ++ indent() << " this.id = id;" << endl << ++ indent() << "}" << endl; ++ ++ f_service_ << indent() << ++ "@java.lang.Override" << endl << ++ indent() << "public T newInstance() {" << endl; ++ indent_up(); ++ ++ f_service_ << indent() << ++ "Object o;" << endl << ++ indent() << "switch (id) {" << endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "case ARG_IN_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << (*f_iter)->get_name() << "_args();" << ++ endl << indent() << " break;" << endl; ++ f_service_ << indent() << ++ "case ARG_OUT_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << (*f_iter)->get_name() << "_result();" << ++ endl << indent() << " break;" << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for (f_iter = functions.begin(); f_iter!= functions.end(); ++f_iter) { ++ f_service_ << indent() << ++ "case ARG_IN_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << extend_service_name << "." << (*f_iter)->get_name() << "_args();" << ++ endl << indent() << " break;" << endl; ++ f_service_ << indent() << ++ "case ARG_OUT_METHOD_" << (*f_iter)->get_name() << ":" << endl << ++ indent() << " o = new " << extend_service_name << "." << (*f_iter)->get_name() << "_result();" << ++ endl << indent() << " break;" << endl; ++ } ++ } ++ ++ f_service_ << indent() << ++ "default:" << endl << indent() << ++ " throw new AssertionError();" << endl << indent() << ++ "}" << endl; ++ ++ f_service_ << indent() << ++ "@java.lang.SuppressWarnings(\"unchecked\")" << endl << ++ indent() << "T t = (T) o;" << endl << indent() << ++ "return t;" << endl; ++ ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl; ++ ++ indent_down(); ++ f_service_ << indent() << "}" << endl; ++} ++ ++void t_java_generator::generate_service_impl_base(t_service* tservice) { ++ f_service_ << ++ indent() << "public static abstract class " << service_name_ << ++ "ImplBase implements " << service_name_ << ", io.grpc.BindableService {" << endl; ++ indent_up(); ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << (*f_iter)->get_name() << ++ "_args request, " << endl << ++ indent() << " io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << ++ "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnimplementedUnaryCall(METHOD_" << (*f_iter)->get_name() << ++ ", responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc" ; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << ++ extend_service_name << "." << (*f_iter)->get_name() << ++ "_args request, " << endl << ++ indent() << " io.grpc.stub.StreamObserver<" << extend_service_name ++ << "." << (*f_iter)->get_name() << "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnimplementedUnaryCall(METHOD_" << (*f_iter)->get_name() << ++ ", responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << ++ " public io.grpc.ServerServiceDefinition bindService() {" << endl; + indent_up(); ++ f_service_ << ++ indent() << "return " << service_name_ << "Grpc.bindService(this);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Abstract Service ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static abstract class Abstract" << service_name_ << ++ " extends " << service_name_ << "ImplBase {}" << endl << endl; ++} ++ ++void t_java_generator::generate_method_descriptors(t_service* tservice) { ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for( f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public static final io.grpc.MethodDescriptor<" << ++ (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << (*f_iter)->get_name() << "_result> METHOD_" << (*f_iter)->get_name() << ++ " = " << endl << indent() << " io.grpc.MethodDescriptor.create(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " io.grpc.MethodDescriptor.MethodType.UNARY," << endl << ++ indent() << " generateFullMethodName(" << "\"" << package_name_ << "." << ++ service_name_ << "\" , \"" << (*f_iter)->get_name() << "\")," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << (*f_iter)->get_name() << ++ "_args>( ARG_IN_METHOD_" << (*f_iter)->get_name() << "))," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << (*f_iter)->get_name() << ++ "_result>( ARG_OUT_METHOD_" << (*f_iter)->get_name() << ")));" << endl << endl; ++ indent_down(); ++ } ++ ++ if(tservice->get_extends() != nullptr) { ++ t_service* extends_service = tservice->get_extends(); ++ functions = extends_service->get_functions(); ++ string extend_service_name = extends_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public static final io.grpc.MethodDescriptor<" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << "_result> METHOD_" ++ << (*f_iter)->get_name() << " = " << endl << indent() << ++ " io.grpc.MethodDescriptor.create(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " io.grpc.MethodDescriptor.MethodType.UNARY," << endl << ++ indent() << " generateFullMethodName(" << "\"" << package_name_ << "." << ++ service_name_ << "\" , \"" << (*f_iter)->get_name() << "\")," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_args>( ARG_IN_METHOD_" << (*f_iter)->get_name() << "))," << endl << ++ indent() << " io.grpc.thrift.ThriftUtils.marshaller(" << endl << ++ indent() << " new ThriftMessageFactory<" << extend_service_name << "." << (*f_iter)->get_name() << ++ "_result>( ARG_OUT_METHOD_" << (*f_iter)->get_name() << ")));" << endl << endl; ++ indent_down(); ++ } ++ } ++} ++ ++void t_java_generator::generate_stub(t_service* tservice) { ++ f_service_ << ++ indent() << ++ "public static " << service_name_ << ++ "Stub newStub(io.grpc.Channel channel) {" << ++ endl; ++ ++ indent_up(); ++ f_service_ << ++ indent() << ++ "return new " << service_name_ << "Stub(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Stub impl ++ ++ f_service_ << ++ indent() << "public static class " << ++ service_name_ << "Stub extends io.grpc.stub.AbstractStub<" << ++ service_name_ << "Stub>" << endl << ++ indent() << " implements " << service_name_ << "{" << endl; ++ indent_up(); ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "Stub(io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "Stub(io.grpc.Channel channel, " << endl << ++ indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "protected " << service_name_ << "Stub build(io.grpc.Channel channel, " << ++ endl << indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "Stub(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << ++ (*f_iter)->get_name() << "_args request," << endl << indent() << ++ " io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << ++ "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request, responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public void " << (*f_iter)->get_name() << "(" << ++ extend_service_name << "." << (*f_iter)->get_name() << "_args request," ++ << endl << indent() << " io.grpc.stub.StreamObserver<" << ++ extend_service_name << "." << (*f_iter)->get_name() << ++ "_result> responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "asyncUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request, responseObserver);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_blocking_stub(t_service* tservice) { ++ f_service_ << ++ indent() << "public static " << service_name_ << ++ "BlockingStub newBlockingStub(" << endl << ++ indent() << " io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "BlockingStub(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Blocking Client ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static interface " << service_name_ << ++ "BlockingClient " ; ++ ++ if (tservice->get_extends() != nullptr) { ++ string extend_service_name = tservice->get_extends()->get_name(); ++ f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << ++ extend_service_name << "BlockingClient " ; ++ } ++ ++ f_service_ << "{" << endl; ++ ++ indent_up(); ++ + vector functions = tservice->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { +- generate_java_doc(f_service_, *f_iter); +- indent(f_service_) << "public " << function_signature(*f_iter) << ";" << endl << endl; ++ f_service_ << ++ indent() << "public " << (*f_iter)->get_name() << "_result " << ++ (*f_iter)->get_name() << "(" << (*f_iter)->get_name() << "_args request);" << endl << endl; ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Blocking Stub impl ++ ++ f_service_ << ++ indent() << "public static class " << ++ service_name_ << "BlockingStub extends io.grpc.stub.AbstractStub<" << ++ service_name_ << "BlockingStub>" << endl << ++ indent() << " implements " << service_name_ << "BlockingClient {"; ++ ++ indent_up(); ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "BlockingStub(io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "BlockingStub(io.grpc.Channel channel, " << endl << ++ indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "protected " << service_name_ << "BlockingStub build(io.grpc.Channel channel, " << ++ endl << indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "BlockingStub(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public " << (*f_iter)->get_name() << "_result " << (*f_iter)->get_name() << "(" << ++ (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return blockingUnaryCall(" << endl << ++ indent() << " getChannel(), METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions(), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public " << extend_service_name << "." << (*f_iter)->get_name() << ++ "_result " << (*f_iter)->get_name() << "(" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return blockingUnaryCall(" << endl << ++ indent() << " getChannel(), METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions(), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_future_stub(t_service* tservice) { ++ f_service_ << ++ indent() << "public static " << service_name_ << ++ "FutureStub newFutureStub(" << endl << ++ indent() << " io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "FutureStub(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Future Client ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static interface " << service_name_ << ++ "FutureClient " ; ++ ++ if (tservice->get_extends() != nullptr) { ++ string extend_service_name = tservice->get_extends()->get_name(); ++ f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << ++ extend_service_name << "FutureClient " ; ++ } ++ f_service_ << "{" << endl; ++ ++ indent_up(); ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ (*f_iter)->get_name() << "_result> " << (*f_iter)->get_name() << "(" << endl << ++ indent() << " " << (*f_iter)->get_name() << "_args request);" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ extend_service_name << "." << (*f_iter)->get_name() << "_result> " << ++ (*f_iter)->get_name() << "(" << endl << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << ++ "_args request);" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // generate Stub impl ++ ++ f_service_ << ++ indent() << "public static class " << ++ service_name_ << "FutureStub extends io.grpc.stub.AbstractStub<" << ++ service_name_ << "FutureStub>" << endl << ++ indent() << " implements " << service_name_ << "FutureClient {" << endl; ++ indent_up(); ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "FutureStub(io.grpc.Channel channel) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "private " << service_name_ << "FutureStub(io.grpc.Channel channel, " << endl << ++ indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "super(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "protected " << service_name_ << "FutureStub build(io.grpc.Channel channel, " << ++ endl << indent() << " io.grpc.CallOptions callOptions) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new " << service_name_ << "FutureStub(channel, callOptions);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ functions = tservice->get_functions(); ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ (*f_iter)->get_name() << "_result> " << (*f_iter)->get_name() << "(" << ++ endl << indent() << " " << (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return futureUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "public com.google.common.util.concurrent.ListenableFuture<" << ++ extend_service_name << "." << (*f_iter)->get_name() << "_result> " << ++ (*f_iter)->get_name() << "(" << endl << indent() << " " << ++ extend_service_name << "." << (*f_iter)->get_name() << "_args request) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return futureUnaryCall(" << endl << ++ indent() << " getChannel().newCall(METHOD_" << (*f_iter)->get_name() << ++ ", getCallOptions()), request);" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ } ++ } ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_method_ids(t_service* tservice) { ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ int i=0; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { ++ f_service_ << ++ indent() << "private static final int METHODID_" << ++ (*f_iter)->get_name() << " = " << i << ";" << endl; ++ } ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { ++ f_service_ << ++ indent() << "private static final int METHODID_" << ++ (*f_iter)->get_name() << " = " << i << ";" << endl; ++ } ++ } ++ f_service_ << endl; ++} ++ ++void t_java_generator::generate_method_handlers(t_service* tservice) { ++ f_service_ << ++ indent() << "private static class MethodHandlers implements" << ++ endl << indent() << " io.grpc.stub.ServerCalls.UnaryMethod," << ++ endl << indent() << " io.grpc.stub.ServerCalls.ServerStreamingMethod," << ++ endl << indent() << " io.grpc.stub.ServerCalls.ClientStreamingMethod," << ++ endl << indent() << " io.grpc.stub.ServerCalls.BidiStreamingMethod {" << ++ endl; ++ indent_up(); ++ f_service_ << ++ indent() << "private final " << service_name_ << " serviceImpl;" << endl << ++ indent() << "private final int methodId;" << endl << endl; ++ ++ f_service_ << ++ indent() << "public MethodHandlers(" << service_name_ << " serviceImpl, int " << ++ "methodId) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "this.serviceImpl = serviceImpl;" << endl << ++ indent() << "this.methodId = methodId;" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // invoke ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "@java.lang.SuppressWarnings(\"unchecked\")" << endl << ++ indent() << "public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) {" << ++ endl; ++ indent_up(); ++ f_service_ << ++ indent() << "switch (methodId) {" << endl; ++ indent_up(); ++ ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "case METHODID_" << (*f_iter)->get_name() << ":" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "serviceImpl." << (*f_iter)->get_name() << "((" << (*f_iter)->get_name() << ++ "_args) request," << endl << ++ indent() << " (io.grpc.stub.StreamObserver<" << (*f_iter)->get_name() << "_result>)" << ++ " responseObserver);" << endl << ++ indent() << "break;" << endl << endl; ++ indent_down(); ++ } ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "case METHODID_" << (*f_iter)->get_name() << ":" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "serviceImpl." << (*f_iter)->get_name() << "((" << extend_service_name << ++ "." << (*f_iter)->get_name() << "_args) request," << endl << ++ indent() << " (io.grpc.stub.StreamObserver<" << extend_service_name << "." << ++ (*f_iter)->get_name() << "_result>)" << " responseObserver);" << endl << ++ indent() << "break;" << endl << endl; ++ indent_down(); ++ } + } ++ f_service_ << ++ indent() << "default:" << endl << ++ indent() << " throw new AssertionError();" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl; ++ indent_down(); ++ f_service_ << ++ indent() << "}" << endl << endl; ++ ++ // invoke ++ f_service_ << ++ indent() << "@java.lang.Override" << endl << ++ indent() << "@java.lang.SuppressWarnings(\"unchecked\")" << endl << ++ indent() << "public io.grpc.stub.StreamObserver invoke(" << endl << ++ indent() << " io.grpc.stub.StreamObserver responseObserver) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "switch (methodId) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "default:" << endl; ++ f_service_ << ++ indent() << " throw new AssertionError();" << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl; + indent_down(); + f_service_ << indent() << "}" << endl << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++ + } + ++void t_java_generator::generate_service_descriptors(t_service* tservice) { ++ // generate service descriptor ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ f_service_ << ++ indent() << "public static io.grpc.ServiceDescriptor getServiceDescriptor() {" << ++ endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return new io.grpc.ServiceDescriptor(SERVICE_NAME"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "," << endl << ++ indent() << " METHOD_" << (*f_iter)->get_name(); ++ } ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << "," << endl << ++ indent() << " METHOD_" << (*f_iter)->get_name(); ++ } ++ } ++ f_service_ << ");" << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++} ++ ++void t_java_generator::generate_service_builder(t_service* tservice) { ++ // bind Service ++ vector functions = tservice->get_functions(); ++ vector::iterator f_iter; ++ f_service_ << ++ indent() << "@java.lang.Deprecated public static io.grpc.ServerServiceDefinition" << ++ " bindService(" << endl << ++ indent() << " final " << service_name_ << " serviceImpl) {" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << "return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor())" << ++ endl; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << " .addMethod(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " METHOD_" << (*f_iter)->get_name() << "," << endl << ++ indent() << " asyncUnaryCall(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " new MethodHandlers<" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " " << (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << (*f_iter)->get_name() << "_result>(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " serviceImpl, METHODID_" << (*f_iter)->get_name() << ")))" << endl; ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ } ++ ++ if (tservice->get_extends() != nullptr) { ++ t_service* extend_service = tservice->get_extends(); ++ functions = extend_service->get_functions(); ++ string extend_service_name = extend_service->get_name() + "Grpc"; ++ for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { ++ f_service_ << ++ indent() << " .addMethod(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " METHOD_" << (*f_iter)->get_name() << "," << endl << ++ indent() << " asyncUnaryCall(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " new MethodHandlers<" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << "_args," << endl << ++ indent() << " " << extend_service_name << "." << (*f_iter)->get_name() << "_result>(" << endl; ++ indent_up(); ++ f_service_ << ++ indent() << " serviceImpl, METHODID_" << (*f_iter)->get_name() << ")))" << endl; ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ indent_down(); ++ } ++ } ++ f_service_ << ++ indent() << " .build();" << endl; ++ indent_down(); ++ f_service_ << indent() << "}" << endl << endl; ++} ++ ++ + void t_java_generator::generate_service_async_interface(t_service* tservice) { + string extends = ""; + string extends_iface = ""; +diff --git a/tutorial/Makefile.am b/tutorial/Makefile.am +index 5865c54..1cffbe6 100755 +--- a/tutorial/Makefile.am ++++ b/tutorial/Makefile.am +@@ -35,11 +35,6 @@ if WITH_D + SUBDIRS += d + endif + +-if WITH_JAVA +-SUBDIRS += java +-SUBDIRS += js +-endif +- + if WITH_PYTHON + SUBDIRS += py + SUBDIRS += py.twisted +@@ -95,4 +90,5 @@ EXTRA_DIST = \ + php \ + shared.thrift \ + tutorial.thrift \ +- README.md ++ README.md \ ++ java +-- +2.8.0.rc3.226.g39d4020 + From a17039aaa443ac8b4f8d5bad51fd6e8c2777071f Mon Sep 17 00:00:00 2001 From: chedeti Date: Mon, 1 Aug 2016 17:32:06 -0700 Subject: [PATCH 14/43] refine code and add README --- Makefile | 1 - build.yaml | 1 - .../grpc++/impl/codegen/thrift_serializer.h | 207 ++++++++++++------ .../impl/codegen/thrift_serializer_inl.h | 169 -------------- include/grpc++/impl/codegen/thrift_utils.h | 10 +- tools/grift/README.md | 15 ++ tools/grift/grpc_plugins_generator.patch | 187 +++++++++++++++- tools/run_tests/sources_and_headers.json | 2 - .../grpc++_test_util/grpc++_test_util.vcxproj | 1 - .../grpc++_test_util.vcxproj.filters | 3 - 10 files changed, 341 insertions(+), 255 deletions(-) delete mode 100644 include/grpc++/impl/codegen/thrift_serializer_inl.h create mode 100644 tools/grift/README.md diff --git a/Makefile b/Makefile index 986c25780f2..31a8448fff6 100644 --- a/Makefile +++ b/Makefile @@ -3924,7 +3924,6 @@ PUBLIC_HEADERS_CXX += \ include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ include/grpc++/impl/codegen/thrift_serializer.h \ - include/grpc++/impl/codegen/thrift_serializer_inl.h \ include/grpc++/impl/codegen/thrift_utils.h \ LIBGRPC++_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_UTIL_SRC)))) diff --git a/build.yaml b/build.yaml index 35af7a6c7a5..02ad6a8842b 100644 --- a/build.yaml +++ b/build.yaml @@ -784,7 +784,6 @@ filegroups: language: c++ public_headers: - include/grpc++/impl/codegen/thrift_serializer.h - - include/grpc++/impl/codegen/thrift_serializer_inl.h - include/grpc++/impl/codegen/thrift_utils.h uses: - grpc++_codegen_base diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index 315d76cf2a5..46112ee5b23 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -36,12 +36,16 @@ #include #include - +#include +#include +#include +#include +#include #include #include +#include #include #include -#include namespace apache { namespace thrift { @@ -49,100 +53,165 @@ namespace util { using apache::thrift::protocol::TBinaryProtocolT; using apache::thrift::protocol::TCompactProtocolT; +using apache::thrift::protocol::TMessageType; using apache::thrift::protocol::TNetworkBigEndian; using apache::thrift::transport::TMemoryBuffer; using apache::thrift::transport::TBufferBase; using apache::thrift::transport::TTransport; -using std::shared_ptr; - -template -class ThriftSerializer { +template class ThriftSerializer { public: ThriftSerializer() - : prepared_ (false) - , lastDeserialized_ (false) - , serializeVersion_ (false) {} - - /** - * Serialize the passed type into the internal buffer - * and returns a pointer to internal buffer and its size - * - */ - template - void serialize(const T& fields, const uint8_t** serializedBuffer, - size_t* serializedLen); - - /** - * Serialize the passed type into the byte buffer - */ - template - void serialize(const T& fields, grpc_byte_buffer** bp); - - /** - * Deserialize the passed char array into the passed type, returns the number - * of bytes that have been consumed from the passed string. - */ - template - uint32_t deserialize(const uint8_t* serializedBuffer, size_t length, - T* fields); - - /** - * Deserialize the passed byte buffer to passed type, returns the number - * of bytes consumed from byte buffer - */ - template - uint32_t deserialize(grpc_byte_buffer* buffer, T* msg); - - void setSerializeVersion(bool value); + : prepared_ (false) + , last_deserialized_ (false) + , serialize_version_ (false) {} virtual ~ThriftSerializer() {} + // Serialize the passed type into the internal buffer + // and returns a pointer to internal buffer and its size + template void Serialize(const T& fields, const uint8_t** serializedBuffer, + size_t* serializedLen) { + // prepare or reset buffer + if (!prepared_ || last_deserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + last_deserialized_ = false; + + // if required serialize protocol version + if (serialize_version_) { + protocol_->writeMessageBegin("", TMessageType(0), 0); + } + + // serilaize fields into buffer + fields.write(protocol_.get()); + + // write the end of message + if (serialize_version_) { + protocol_->writeMessageEnd(); + } + + uint8_t* byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + *serializedBuffer = byteBuffer; + *serializedLen = byteBufferSize; + } + + // Serialize the passed type into the byte buffer + template void Serialize(const T& fields, grpc_byte_buffer** bp) { + + const uint8_t* byteBuffer; + size_t byteBufferSize; + + Serialize(fields, &byteBuffer, &byteBufferSize); + + gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + + *bp = grpc_raw_byte_buffer_create(&slice, 1); - /** - * Set the container size limit to deserialize - * This function should be called after buffer_ is initialized - */ - void setContainerSizeLimit(int32_t container_limit) { + gpr_slice_unref(slice); + } + + // Deserialize the passed char array into the passed type, returns the number + // of bytes that have been consumed from the passed string. + template uint32_t Deserialize(const uint8_t* serializedBuffer, size_t length, + T* fields) { + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + last_deserialized_ = true; + + //reset buffer transport + buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + + // read the protocol version if necessary + if (serialize_version_) { + std::string name = ""; + TMessageType mt = (TMessageType) 0; + int32_t seq_id = 0; + protocol_->readMessageBegin(name, mt, seq_id); + } + + // deserialize buffer into fields + uint32_t len = fields->read(protocol_.get()); + + // read the end of message + if (serialize_version_) { + protocol_->readMessageEnd(); + } + + return len; + } + + + // Deserialize the passed byte buffer to passed type, returns the number + // of bytes consumed from byte buffer + template uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { + + grpc_byte_buffer_reader reader; + grpc_byte_buffer_reader_init(&reader, buffer); + + gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); + + uint32_t len = Deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), msg); + + gpr_slice_unref(slice); + + grpc_byte_buffer_reader_destroy(&reader); + + return len; + } + + // set serialization version flag + void SetSerializeVersion(bool value) { + serialize_version_ = value; + } + + // Set the container size limit to deserialize + // This function should be called after buffer_ is initialized + void SetContainerSizeLimit(int32_t container_limit) { if (!prepared_) { prepare(); } protocol_->setContainerSizeLimit(container_limit); } - /** - * Set the string size limit to deserialize - * This function should be called after buffer_ is initialized - */ - void setStringSizeLimit(int32_t string_limit) { + // Set the string size limit to deserialize + // This function should be called after buffer_ is initialized + void SetStringSizeLimit(int32_t string_limit) { if (!prepared_) { prepare(); } protocol_->setStringSizeLimit(string_limit); } +private: + bool prepared_; + bool last_deserialized_; + boost::shared_ptr buffer_; + std::shared_ptr protocol_; + bool serialize_version_; - private: - void prepare(); - - private: - typedef P Protocol; - bool prepared_; - bool lastDeserialized_; - boost::shared_ptr buffer_; - shared_ptr protocol_; - bool serializeVersion_; -}; // ThriftSerializer + void prepare() { + buffer_.reset(new TMemoryBuffer()); -template -struct ThriftSerializerBinary : public ThriftSerializer > {}; + // create a protocol for the memory buffer transport + protocol_.reset(new Protocol(buffer_)); + prepared_ = true; + } -template -struct ThriftSerializerCompact : public ThriftSerializer >{ }; +}; // ThriftSerializer -}}} // namespace apache::thrift::util +typedef ThriftSerializer> ThriftSerializerBinary; +typedef ThriftSerializer> ThriftSerializerCompact; -#include +} // namespace util +} // namespace thrift +} // namespace apache -#endif +#endif \ No newline at end of file diff --git a/include/grpc++/impl/codegen/thrift_serializer_inl.h b/include/grpc++/impl/codegen/thrift_serializer_inl.h deleted file mode 100644 index 866ecf6312d..00000000000 --- a/include/grpc++/impl/codegen/thrift_serializer_inl.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * - * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - - #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H - #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H - -#include -#include -#include -#include -#include -#include -#include - -namespace apache { -namespace thrift { -namespace util { - -using apache::thrift::protocol::TMessageType; - -template -template -void ThriftSerializer::serialize(const T& fields, - const uint8_t** serializedBuffer, size_t* serializedLen) { - - // prepare or reset buffer - if (!prepared_ || lastDeserialized_) { - prepare(); - } else { - buffer_->resetBuffer(); - } - lastDeserialized_ = false; - - // if required serialize protocol version - if (serializeVersion_) { - protocol_->writeMessageBegin("", TMessageType(0), 0); - } - - // serilaize fields into buffer - fields.write(protocol_.get()); - - // write the end of message - if (serializeVersion_) { - protocol_->writeMessageEnd(); - } - - // assign buffer to string - uint8_t* byteBuffer; - uint32_t byteBufferSize; - buffer_->getBuffer(&byteBuffer, &byteBufferSize); - *serializedBuffer = byteBuffer; - *serializedLen = byteBufferSize; -} - -template -template -void ThriftSerializer::serialize(const T& fields, grpc_byte_buffer** bp) { - - const uint8_t* byteBuffer; - size_t byteBufferSize; - serialize(fields, &byteBuffer, &byteBufferSize); - - gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); - - *bp = grpc_raw_byte_buffer_create(&slice, 1); - - gpr_slice_unref(slice); -} - -template -template -uint32_t ThriftSerializer::deserialize(const uint8_t* serializedBuffer, - size_t length, T* fields) { - // prepare buffer if necessary - if (!prepared_) { - prepare(); - } - lastDeserialized_ = true; - - //reset buffer transport - buffer_->resetBuffer((uint8_t*)serializedBuffer, length); - - // read the protocol version if necessary - if (serializeVersion_) { - std::string name = ""; - TMessageType mt = (TMessageType) 0; - int32_t seq_id = 0; - protocol_->readMessageBegin(name, mt, seq_id); - } - - // deserialize buffer into fields - uint32_t len = fields->read(protocol_.get()); - - // read the end of message - if (serializeVersion_) { - protocol_->readMessageEnd(); - } - - return len; -} - -template -template -uint32_t ThriftSerializer::deserialize(grpc_byte_buffer* bp, T* fields) { - grpc_byte_buffer_reader reader; - grpc_byte_buffer_reader_init(&reader, bp); - - gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); - - uint32_t len = deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), fields); - - gpr_slice_unref(slice); - - grpc_byte_buffer_reader_destroy(&reader); - - return len; -} - -template -void ThriftSerializer::setSerializeVersion(bool value) { - serializeVersion_ = value; -} - -template -void -ThriftSerializer::prepare() -{ - - buffer_.reset(new TMemoryBuffer()); - - // create a protocol for the memory buffer transport - protocol_.reset(new Protocol(buffer_)); - - prepared_ = true; -} - -}}} // namespace apache::thrift::util - -#endif diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h index 629441149fd..14332c05219 100644 --- a/include/grpc++/impl/codegen/thrift_utils.h +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -44,9 +44,7 @@ #include #include #include -#include #include -#include #include namespace grpc { @@ -62,9 +60,9 @@ class SerializationTraits serializer; + ThriftSerializerCompact serializer; - serializer.serialize(msg, bp); + serializer.Serialize(msg, bp); return Status(StatusCode::OK, "ok"); } @@ -76,8 +74,8 @@ class SerializationTraits deserializer; - deserializer.deserialize(buffer, msg); + ThriftSerializerCompact deserializer; + deserializer.Deserialize(buffer, msg); grpc_byte_buffer_destroy(buffer); diff --git a/tools/grift/README.md b/tools/grift/README.md new file mode 100644 index 00000000000..a4cb87bff1a --- /dev/null +++ b/tools/grift/README.md @@ -0,0 +1,15 @@ +Copyright 2016 Google Inc. + +#Documentation + +grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with GRPC. + +This integration allows you to use grpc to send thrift messages in C++ and java. + +By default grift uses Compact Protocol to serialize thrift messages. + +#Installation + +Before Installing thrift make sure to apply this [patch](grpc_plugins_generate.patch) to third_party/thrift. +Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to +install thrift. \ No newline at end of file diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index 2779dfb59eb..9c18db35995 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -1,7 +1,7 @@ From 0894590b5020c38106d4ebb2291994668c64f9dd Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:47:47 -0700 -Subject: [PATCH 1/3] don't build tests +Subject: [PATCH 1/5] don't build tests --- Makefile.am | 7 ++----- @@ -62,7 +62,7 @@ index 6fd15d2..7de1fad 100755 From 04244fa7805740761db757e4c44251f723d85839 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 -Subject: [PATCH 2/3] grpc cpp plugins generator with example +Subject: [PATCH 2/5] grpc cpp plugins generator with example --- compiler/cpp/src/generate/t_cpp_generator.cc | 476 +++++++++++++++++++++++---- @@ -1401,7 +1401,7 @@ index 0000000..de3c9a4 From 1d47ed062e62d136c3db9f6fc1dde9e2f794cf22 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 -Subject: [PATCH 3/3] grpc java plugins generator +Subject: [PATCH 3/5] grpc java plugins generator for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thrift --- @@ -2415,3 +2415,184 @@ index 5865c54..1cffbe6 100755 -- 2.8.0.rc3.226.g39d4020 + +From a9769a0fa08f553da76215ca59a8fd797b92a853 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Mon, 1 Aug 2016 16:56:36 -0700 +Subject: [PATCH 4/5] maintain consistency with grpc + +--- + compiler/cpp/src/generate/t_cpp_generator.cc | 20 ++++++++++---------- + tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} | 0 + tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} | 0 + tutorial/cpp/Makefile.am | 8 ++++---- + 4 files changed, 14 insertions(+), 14 deletions(-) + rename tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} (100%) + rename tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} (100%) + +diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc +index 9c3399b..4e00129 100644 +--- a/compiler/cpp/src/generate/t_cpp_generator.cc ++++ b/compiler/cpp/src/generate/t_cpp_generator.cc +@@ -1641,7 +1641,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { + << endl; + + t_service* extends_service = tservice->get_extends(); +- if (extends_service != nullptr) { ++ if (extends_service) { + f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) + << extends_service->get_name() << ".grpc.thrift.h\"" << endl; + } +@@ -1733,7 +1733,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { + indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + +@@ -1749,9 +1749,9 @@ void t_cpp_generator::generate_service(t_service* tservice) { + "};" << endl; + + // Generate service class +- if ( extends_service != nullptr ) { ++ if ( extends_service) { + f_header_ << "class " << service_name_ << " : public " << +- extends_service->get_name() << " {" << endl << ++ type_name(extends_service) << " {" << endl << + "public:" << endl; + } + else { +@@ -1841,7 +1841,7 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { + void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { + + string extends = ""; +- if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; + } + +@@ -1890,7 +1890,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + } + + t_service* extends_service = tservice->get_extends(); +- if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; +@@ -1914,7 +1914,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; +@@ -1944,7 +1944,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; +@@ -2002,7 +2002,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { + + } + +- if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { +@@ -2049,7 +2049,7 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + } + + string extends = ""; +- if (tservice->get_extends() != NULL) { ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; + if (style == "CobCl" && gen_templates_) { + // TODO(simpkins): If gen_templates_ is enabled, we currently assume all +diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/GrpcClient.cpp +similarity index 100% +rename from tutorial/cpp/CppClient.cpp +rename to tutorial/cpp/GrpcClient.cpp +diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/GrpcServer.cpp +similarity index 100% +rename from tutorial/cpp/CppServer.cpp +rename to tutorial/cpp/GrpcServer.cpp +diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am +index 581f75e..39d85e1 100755 +--- a/tutorial/cpp/Makefile.am ++++ b/tutorial/cpp/Makefile.am +@@ -39,14 +39,14 @@ noinst_PROGRAMS = \ + TestClient + + TestServer_SOURCES = \ +- CppServer.cpp ++ GrpcServer.cpp + + TestServer_LDADD = \ + libtestgencpp.la \ + $(top_builddir)/lib/cpp/libthrift.la + + TestClient_SOURCES = \ +- CppClient.cpp ++ GrpcClient.cpp + + TestClient_LDADD = \ + libtestgencpp.la \ +@@ -78,5 +78,5 @@ style-local: + + EXTRA_DIST = \ + CMakeLists.txt \ +- CppClient.cpp \ +- CppServer.cpp ++ GrpcClient.cpp \ ++ GrpcServer.cpp +-- +2.8.0.rc3.226.g39d4020 + + +From b4bc0c810f00a1b86516774306ff4017e3d2d252 Mon Sep 17 00:00:00 2001 +From: chedeti +Date: Mon, 1 Aug 2016 17:00:17 -0700 +Subject: [PATCH 5/5] fix typo + +--- + tutorial/cpp/GrpcClient.cpp | 2 +- + tutorial/cpp/GrpcServer.cpp | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp +index c41604e..41a7acf 100644 +--- a/tutorial/cpp/GrpcClient.cpp ++++ b/tutorial/cpp/GrpcClient.cpp +@@ -1,6 +1,6 @@ + /* + * +- * Copyright 2015, Google Inc. ++ * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without +diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp +index c838b61..f63db57 100644 +--- a/tutorial/cpp/GrpcServer.cpp ++++ b/tutorial/cpp/GrpcServer.cpp +@@ -1,6 +1,6 @@ + /* + * +- * Copyright 2015, Google Inc. ++ * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without +-- +2.8.0.rc3.226.g39d4020 + diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 06ac86fc9c4..82b443a4742 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -6792,14 +6792,12 @@ ], "headers": [ "include/grpc++/impl/codegen/thrift_serializer.h", - "include/grpc++/impl/codegen/thrift_serializer_inl.h", "include/grpc++/impl/codegen/thrift_utils.h" ], "language": "c++", "name": "thrift_util", "src": [ "include/grpc++/impl/codegen/thrift_serializer.h", - "include/grpc++/impl/codegen/thrift_serializer_inl.h", "include/grpc++/impl/codegen/thrift_utils.h" ], "third_party": false, diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj index 96ce3b89164..c2c7d00a6d9 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj @@ -201,7 +201,6 @@ - diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters index 2105e672df9..9b8c8ddfada 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters @@ -195,9 +195,6 @@ include\grpc++\impl\codegen - - include\grpc++\impl\codegen - include\grpc++\impl\codegen From f5dcc9b5b5d1ade84eb7bac07abbac56f5f86675 Mon Sep 17 00:00:00 2001 From: chedeti Date: Mon, 1 Aug 2016 17:57:24 -0700 Subject: [PATCH 15/43] fix typos --- tools/grift/README.md | 16 ++++++++++++++-- tools/grift/grpc_plugins_generator.patch | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tools/grift/README.md b/tools/grift/README.md index a4cb87bff1a..25c67451326 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -6,10 +6,22 @@ grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Se This integration allows you to use grpc to send thrift messages in C++ and java. -By default grift uses Compact Protocol to serialize thrift messages. +grift uses Compact Protocol to serialize thrift messages. + +##generating grpc plugins for thrift services + +###CPP +```sh + $ thrift --gen cpp +``` + +###JAVA +```sh + $ thrift --gen java +``` #Installation -Before Installing thrift make sure to apply this [patch](grpc_plugins_generate.patch) to third_party/thrift. +Before Installing thrift make sure to apply this [patch](grpc_plugins_generator.patch) to third_party/thrift. Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to install thrift. \ No newline at end of file diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index 9c18db35995..eeee1612519 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -2559,18 +2559,18 @@ index 581f75e..39d85e1 100755 2.8.0.rc3.226.g39d4020 -From b4bc0c810f00a1b86516774306ff4017e3d2d252 Mon Sep 17 00:00:00 2001 +From bc74fca1ee73333819724f51d5aaff3546443ed0 Mon Sep 17 00:00:00 2001 From: chedeti Date: Mon, 1 Aug 2016 17:00:17 -0700 Subject: [PATCH 5/5] fix typo --- - tutorial/cpp/GrpcClient.cpp | 2 +- + tutorial/cpp/GrpcClient.cpp | 4 ++-- tutorial/cpp/GrpcServer.cpp | 2 +- - 2 files changed, 2 insertions(+), 2 deletions(-) + 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp -index c41604e..41a7acf 100644 +index c41604e..ab1fe77 100644 --- a/tutorial/cpp/GrpcClient.cpp +++ b/tutorial/cpp/GrpcClient.cpp @@ -1,6 +1,6 @@ @@ -2581,6 +2581,15 @@ index c41604e..41a7acf 100644 * All rights reserved. * * Redistribution and use in source and binary forms, with or without +@@ -50,7 +50,7 @@ class GreeterClient { + GreeterClient(std::shared_ptr channel) + : stub_(Greeter::NewStub(channel)) {} + +- // Assambles the client's payload, sends it and presents the response back ++ // Assembles the client's payload, sends it and presents the response back + // from the server. + std::string SayHello(const std::string& user) { + // Data we are sending to the server. diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp index c838b61..f63db57 100644 --- a/tutorial/cpp/GrpcServer.cpp From 00be9de95cf040e2705c137ea213140562f7571d Mon Sep 17 00:00:00 2001 From: chedeti Date: Tue, 2 Aug 2016 10:44:41 -0700 Subject: [PATCH 16/43] fix codegen patch --- tools/grift/grpc_plugins_generator.patch | 880 ++++++++++------------- 1 file changed, 381 insertions(+), 499 deletions(-) diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index eeee1612519..3a6710c224e 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -1,7 +1,7 @@ From 0894590b5020c38106d4ebb2291994668c64f9dd Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:47:47 -0700 -Subject: [PATCH 1/5] don't build tests +Subject: [PATCH 1/3] don't build tests --- Makefile.am | 7 ++----- @@ -59,24 +59,30 @@ index 6fd15d2..7de1fad 100755 2.8.0.rc3.226.g39d4020 -From 04244fa7805740761db757e4c44251f723d85839 Mon Sep 17 00:00:00 2001 +From c8577ad5513543c57a81ad1bf4927cc8a78baa03 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 -Subject: [PATCH 2/5] grpc cpp plugins generator with example +Subject: [PATCH 2/3] grpc cpp plugins generator with example --- - compiler/cpp/src/generate/t_cpp_generator.cc | 476 +++++++++++++++++++++++---- + compiler/cpp/src/generate/t_cpp_generator.cc | 478 +++++++++++++++++++++++---- tutorial/cpp/CMakeLists.txt | 53 --- - tutorial/cpp/CppClient.cpp | 134 ++++---- - tutorial/cpp/CppServer.cpp | 226 ++++--------- - tutorial/cpp/Makefile.am | 58 ++-- + tutorial/cpp/CppClient.cpp | 80 ----- + tutorial/cpp/CppServer.cpp | 181 ---------- + tutorial/cpp/GrpcClient.cpp | 94 ++++++ + tutorial/cpp/GrpcServer.cpp | 87 +++++ + tutorial/cpp/Makefile.am | 66 ++-- tutorial/cpp/test.thrift | 13 + - 6 files changed, 590 insertions(+), 370 deletions(-) + 8 files changed, 636 insertions(+), 416 deletions(-) delete mode 100644 tutorial/cpp/CMakeLists.txt + delete mode 100644 tutorial/cpp/CppClient.cpp + delete mode 100644 tutorial/cpp/CppServer.cpp + create mode 100644 tutorial/cpp/GrpcClient.cpp + create mode 100644 tutorial/cpp/GrpcServer.cpp create mode 100644 tutorial/cpp/test.thrift diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc -index 6c04899..9c3399b 100644 +index 6c04899..4e00129 100644 --- a/compiler/cpp/src/generate/t_cpp_generator.cc +++ b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -162,6 +162,8 @@ public: @@ -266,7 +272,7 @@ index 6c04899..9c3399b 100644 t_service* extends_service = tservice->get_extends(); - if (extends_service != NULL) { -+ if (extends_service != nullptr) { ++ if (extends_service) { f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) - << extends_service->get_name() << ".h\"" << endl; + << extends_service->get_name() << ".grpc.thrift.h\"" << endl; @@ -355,7 +361,7 @@ index 6c04899..9c3399b 100644 + indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + @@ -371,9 +377,9 @@ index 6c04899..9c3399b 100644 + "};" << endl; + + // Generate service class -+ if ( extends_service != nullptr ) { ++ if ( extends_service) { + f_header_ << "class " << service_name_ << " : public " << -+ extends_service->get_name() << " {" << endl << ++ type_name(extends_service) << " {" << endl << + "public:" << endl; + } + else { @@ -442,7 +448,7 @@ index 6c04899..9c3399b 100644 +void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { + + string extends = ""; -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; + } + @@ -491,7 +497,7 @@ index 6c04899..9c3399b 100644 + } + + t_service* extends_service = tservice->get_extends(); -+ if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; @@ -515,7 +521,7 @@ index 6c04899..9c3399b 100644 + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; @@ -545,7 +551,7 @@ index 6c04899..9c3399b 100644 + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + // generate inherited methods + vector functions = extends_service->get_functions(); + vector::iterator f_iter; @@ -603,7 +609,7 @@ index 6c04899..9c3399b 100644 + + } + -+ if (extends_service != nullptr) { ++ if (extends_service) { + vector functions = extends_service->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { @@ -642,11 +648,13 @@ index 6c04899..9c3399b 100644 if (style == "CobCl") { // Forward declare the client. string client_name = service_name_ + "CobClient"; -@@ -1765,12 +2050,14 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1764,13 +2049,15 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty + } string extends = ""; - if (tservice->get_extends() != NULL) { +- if (tservice->get_extends() != NULL) { - extends = " : virtual public " + type_name(tservice->get_extends()) + style + "If"; ++ if (tservice->get_extends()) { + extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; if (style == "CobCl" && gen_templates_) { // TODO(simpkins): If gen_templates_ is enabled, we currently assume all @@ -867,11 +875,12 @@ index 8a3d085..0000000 -LINK_AGAINST_THRIFT_LIBRARY(TutorialClient thrift) -target_link_libraries(TutorialClient ${ZLIB_LIBRARIES}) diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/CppClient.cpp -index 2763fee..c41604e 100644 +deleted file mode 100644 +index 2763fee..0000000 --- a/tutorial/cpp/CppClient.cpp -+++ b/tutorial/cpp/CppClient.cpp -@@ -1,80 +1,94 @@ - /* ++++ /dev/null +@@ -1,80 +0,0 @@ +-/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information @@ -879,9 +888,274 @@ index 2763fee..c41604e 100644 - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * +- * - * http://www.apache.org/licenses/LICENSE-2.0 -+ * Copyright 2015, Google Inc. +- * +- * Unless required by applicable law or agreed to in writing, +- * software distributed under the License is distributed on an +- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +- * KIND, either express or implied. See the License for the +- * specific language governing permissions and limitations +- * under the License. +- */ +- +-#include +- +-#include +-#include +-#include +- +-#include "../gen-cpp/Calculator.h" +- +-using namespace std; +-using namespace apache::thrift; +-using namespace apache::thrift::protocol; +-using namespace apache::thrift::transport; +- +-using namespace tutorial; +-using namespace shared; +- +-int main() { +- boost::shared_ptr socket(new TSocket("localhost", 9090)); +- boost::shared_ptr transport(new TBufferedTransport(socket)); +- boost::shared_ptr protocol(new TBinaryProtocol(transport)); +- CalculatorClient client(protocol); +- +- try { +- transport->open(); +- +- client.ping(); +- cout << "ping()" << endl; +- +- cout << "1 + 1 = " << client.add(1, 1) << endl; +- +- Work work; +- work.op = Operation::DIVIDE; +- work.num1 = 1; +- work.num2 = 0; +- +- try { +- client.calculate(1, work); +- cout << "Whoa? We can divide by zero!" << endl; +- } catch (InvalidOperation& io) { +- cout << "InvalidOperation: " << io.why << endl; +- // or using generated operator<<: cout << io << endl; +- // or by using std::exception native method what(): cout << io.what() << endl; +- } +- +- work.op = Operation::SUBTRACT; +- work.num1 = 15; +- work.num2 = 10; +- int32_t diff = client.calculate(1, work); +- cout << "15 - 10 = " << diff << endl; +- +- // Note that C++ uses return by reference for complex types to avoid +- // costly copy construction +- SharedStruct ss; +- client.getStruct(ss, 1); +- cout << "Received log: " << ss << endl; +- +- transport->close(); +- } catch (TException& tx) { +- cout << "ERROR: " << tx.what() << endl; +- } +-} +diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/CppServer.cpp +deleted file mode 100644 +index eafffa9..0000000 +--- a/tutorial/cpp/CppServer.cpp ++++ /dev/null +@@ -1,181 +0,0 @@ +-/* +- * Licensed to the Apache Software Foundation (ASF) under one +- * or more contributor license agreements. See the NOTICE file +- * distributed with this work for additional information +- * regarding copyright ownership. The ASF licenses this file +- * to you under the Apache License, Version 2.0 (the +- * "License"); you may not use this file except in compliance +- * with the License. You may obtain a copy of the License at +- * +- * http://www.apache.org/licenses/LICENSE-2.0 +- * +- * Unless required by applicable law or agreed to in writing, +- * software distributed under the License is distributed on an +- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +- * KIND, either express or implied. See the License for the +- * specific language governing permissions and limitations +- * under the License. +- */ +- +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +-#include +- +-#include +- +-#include +-#include +-#include +- +-#include "../gen-cpp/Calculator.h" +- +-using namespace std; +-using namespace apache::thrift; +-using namespace apache::thrift::concurrency; +-using namespace apache::thrift::protocol; +-using namespace apache::thrift::transport; +-using namespace apache::thrift::server; +- +-using namespace tutorial; +-using namespace shared; +- +-class CalculatorHandler : public CalculatorIf { +-public: +- CalculatorHandler() {} +- +- void ping() { cout << "ping()" << endl; } +- +- int32_t add(const int32_t n1, const int32_t n2) { +- cout << "add(" << n1 << ", " << n2 << ")" << endl; +- return n1 + n2; +- } +- +- int32_t calculate(const int32_t logid, const Work& work) { +- cout << "calculate(" << logid << ", " << work << ")" << endl; +- int32_t val; +- +- switch (work.op) { +- case Operation::ADD: +- val = work.num1 + work.num2; +- break; +- case Operation::SUBTRACT: +- val = work.num1 - work.num2; +- break; +- case Operation::MULTIPLY: +- val = work.num1 * work.num2; +- break; +- case Operation::DIVIDE: +- if (work.num2 == 0) { +- InvalidOperation io; +- io.whatOp = work.op; +- io.why = "Cannot divide by 0"; +- throw io; +- } +- val = work.num1 / work.num2; +- break; +- default: +- InvalidOperation io; +- io.whatOp = work.op; +- io.why = "Invalid Operation"; +- throw io; +- } +- +- SharedStruct ss; +- ss.key = logid; +- ss.value = to_string(val); +- +- log[logid] = ss; +- +- return val; +- } +- +- void getStruct(SharedStruct& ret, const int32_t logid) { +- cout << "getStruct(" << logid << ")" << endl; +- ret = log[logid]; +- } +- +- void zip() { cout << "zip()" << endl; } +- +-protected: +- map log; +-}; +- +-/* +- CalculatorIfFactory is code generated. +- CalculatorCloneFactory is useful for getting access to the server side of the +- transport. It is also useful for making per-connection state. Without this +- CloneFactory, all connections will end up sharing the same handler instance. +-*/ +-class CalculatorCloneFactory : virtual public CalculatorIfFactory { +- public: +- virtual ~CalculatorCloneFactory() {} +- virtual CalculatorIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) +- { +- boost::shared_ptr sock = boost::dynamic_pointer_cast(connInfo.transport); +- cout << "Incoming connection\n"; +- cout << "\tSocketInfo: " << sock->getSocketInfo() << "\n"; +- cout << "\tPeerHost: " << sock->getPeerHost() << "\n"; +- cout << "\tPeerAddress: " << sock->getPeerAddress() << "\n"; +- cout << "\tPeerPort: " << sock->getPeerPort() << "\n"; +- return new CalculatorHandler; +- } +- virtual void releaseHandler( ::shared::SharedServiceIf* handler) { +- delete handler; +- } +-}; +- +-int main() { +- TThreadedServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), //port +- boost::make_shared(), +- boost::make_shared()); +- +- /* +- // if you don't need per-connection state, do the following instead +- TThreadedServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), //port +- boost::make_shared(), +- boost::make_shared()); +- */ +- +- /** +- * Here are some alternate server types... +- +- // This server only allows one connection at a time, but spawns no threads +- TSimpleServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), +- boost::make_shared(), +- boost::make_shared()); +- +- const int workerCount = 4; +- +- boost::shared_ptr threadManager = +- ThreadManager::newSimpleThreadManager(workerCount); +- threadManager->threadFactory( +- boost::make_shared()); +- threadManager->start(); +- +- // This server allows "workerCount" connection at a time, and reuses threads +- TThreadPoolServer server( +- boost::make_shared(boost::make_shared()), +- boost::make_shared(9090), +- boost::make_shared(), +- boost::make_shared(), +- threadManager); +- */ +- +- cout << "Starting the server..." << endl; +- server.serve(); +- cout << "Done." << endl; +- return 0; +-} +diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp +new file mode 100644 +index 0000000..ab1fe77 +--- /dev/null ++++ b/tutorial/cpp/GrpcClient.cpp +@@ -0,0 +1,94 @@ ++/* ++ * ++ * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without @@ -909,105 +1183,57 @@ index 2763fee..c41604e 100644 + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * -- * Unless required by applicable law or agreed to in writing, -- * software distributed under the License is distributed on an -- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -- * KIND, either express or implied. See the License for the -- * specific language governing permissions and limitations -- * under the License. - */ - - #include ++ * ++ */ ++ ++#include +#include +#include - --#include --#include --#include -- --#include "../gen-cpp/Calculator.h" ++ +#include - --using namespace std; --using namespace apache::thrift; --using namespace apache::thrift::protocol; --using namespace apache::thrift::transport; ++ +#include "gen-cpp/Greeter.grpc.thrift.h" - --using namespace tutorial; --using namespace shared; ++ +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; +using test::Greeter; +using namespace test; - --int main() { -- boost::shared_ptr socket(new TSocket("localhost", 9090)); -- boost::shared_ptr transport(new TBufferedTransport(socket)); -- boost::shared_ptr protocol(new TBinaryProtocol(transport)); -- CalculatorClient client(protocol); ++ +class GreeterClient { + public: + GreeterClient(std::shared_ptr channel) + : stub_(Greeter::NewStub(channel)) {} - -- try { -- transport->open(); -+ // Assambles the client's payload, sends it and presents the response back ++ ++ // Assembles the client's payload, sends it and presents the response back + // from the server. + std::string SayHello(const std::string& user) { + // Data we are sending to the server. + Greeter::SayHelloReq req; + req.request.name = user; - -- client.ping(); -- cout << "ping()" << endl; ++ + // Container for the data we expect from the server. + Greeter::SayHelloResp reply; - -- cout << "1 + 1 = " << client.add(1, 1) << endl; ++ + // Context for the client. It could be used to convey extra information to + // the server and/or tweak certain RPC behaviors. + ClientContext context; - -- Work work; -- work.op = Operation::DIVIDE; -- work.num1 = 1; -- work.num2 = 0; ++ + // The actual RPC. + Status status = stub_->SayHello(&context, req, &reply); - -- try { -- client.calculate(1, work); -- cout << "Whoa? We can divide by zero!" << endl; -- } catch (InvalidOperation& io) { -- cout << "InvalidOperation: " << io.why << endl; -- // or using generated operator<<: cout << io << endl; -- // or by using std::exception native method what(): cout << io.what() << endl; ++ + // Act upon its status. + if (status.ok()) { + return reply.success.message; + } else { + return "RPC failed"; - } ++ } + } - -- work.op = Operation::SUBTRACT; -- work.num1 = 15; -- work.num2 = 10; -- int32_t diff = client.calculate(1, work); -- cout << "15 - 10 = " << diff << endl; ++ + private: + std::unique_ptr stub_; +}; - -- // Note that C++ uses return by reference for complex types to avoid -- // costly copy construction -- SharedStruct ss; -- client.getStruct(ss, 1); -- cout << "Received log: " << ss << endl; ++ +int main() { + // Instantiate the client. It requires a channel, out of which the actual RPCs + // are created. This channel models a connection to an endpoint (in this case, @@ -1018,29 +1244,18 @@ index 2763fee..c41604e 100644 + std::string user("world"); + std::string reply = greeter.SayHello(user); + std::cout << "Greeter received: " << reply << std::endl; - -- transport->close(); -- } catch (TException& tx) { -- cout << "ERROR: " << tx.what() << endl; -- } ++ + return 0; - } -diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/CppServer.cpp -index eafffa9..c838b61 100644 ---- a/tutorial/cpp/CppServer.cpp -+++ b/tutorial/cpp/CppServer.cpp -@@ -1,181 +1,87 @@ - /* -- * Licensed to the Apache Software Foundation (ASF) under one -- * or more contributor license agreements. See the NOTICE file -- * distributed with this work for additional information -- * regarding copyright ownership. The ASF licenses this file -- * to you under the Apache License, Version 2.0 (the -- * "License"); you may not use this file except in compliance -- * with the License. You may obtain a copy of the License at - * -- * http://www.apache.org/licenses/LICENSE-2.0 -+ * Copyright 2015, Google Inc. ++} +diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp +new file mode 100644 +index 0000000..f63db57 +--- /dev/null ++++ b/tutorial/cpp/GrpcServer.cpp +@@ -0,0 +1,87 @@ ++/* ++ * ++ * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without @@ -1068,147 +1283,39 @@ index eafffa9..c838b61 100644 + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * -- * Unless required by applicable law or agreed to in writing, -- * software distributed under the License is distributed on an -- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -- * KIND, either express or implied. See the License for the -- * specific language governing permissions and limitations -- * under the License. - */ - --#include --#include --#include --#include --#include --#include --#include --#include --#include --#include -- --#include -- - #include --#include --#include -- --#include "../gen-cpp/Calculator.h" ++ * ++ */ ++ ++#include +#include +#include - --using namespace std; --using namespace apache::thrift; --using namespace apache::thrift::concurrency; --using namespace apache::thrift::protocol; --using namespace apache::thrift::transport; --using namespace apache::thrift::server; ++ +#include - --using namespace tutorial; --using namespace shared; -+#include "gen-cpp/Greeter.grpc.thrift.h" -+#include - --class CalculatorHandler : public CalculatorIf { --public: -- CalculatorHandler() {} ++ ++#include "gen-cpp/Greeter.grpc.thrift.h" ++#include ++ +using grpc::Server; +using grpc::ServerBuilder; +using grpc::ServerContext; +using grpc::Status; +using test::Greeter; - -- void ping() { cout << "ping()" << endl; } ++ +using namespace grpc; +using namespace test; - -- int32_t add(const int32_t n1, const int32_t n2) { -- cout << "add(" << n1 << ", " << n2 << ")" << endl; -- return n1 + n2; -- } ++ +// Logic and data behind the server's behavior. +class GreeterServiceImpl final : public Greeter::Service { + Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, + Greeter::SayHelloResp* reply) override { + std::string prefix("Hello "); - -- int32_t calculate(const int32_t logid, const Work& work) { -- cout << "calculate(" << logid << ", " << work << ")" << endl; -- int32_t val; ++ + reply->success.message = prefix + request->request.name; - -- switch (work.op) { -- case Operation::ADD: -- val = work.num1 + work.num2; -- break; -- case Operation::SUBTRACT: -- val = work.num1 - work.num2; -- break; -- case Operation::MULTIPLY: -- val = work.num1 * work.num2; -- break; -- case Operation::DIVIDE: -- if (work.num2 == 0) { -- InvalidOperation io; -- io.whatOp = work.op; -- io.why = "Cannot divide by 0"; -- throw io; -- } -- val = work.num1 / work.num2; -- break; -- default: -- InvalidOperation io; -- io.whatOp = work.op; -- io.why = "Invalid Operation"; -- throw io; -- } -- -- SharedStruct ss; -- ss.key = logid; -- ss.value = to_string(val); -- -- log[logid] = ss; -- -- return val; ++ + return Status::OK; - } -- -- void getStruct(SharedStruct& ret, const int32_t logid) { -- cout << "getStruct(" << logid << ")" << endl; -- ret = log[logid]; -- } -- -- void zip() { cout << "zip()" << endl; } -- --protected: -- map log; - }; - --/* -- CalculatorIfFactory is code generated. -- CalculatorCloneFactory is useful for getting access to the server side of the -- transport. It is also useful for making per-connection state. Without this -- CloneFactory, all connections will end up sharing the same handler instance. --*/ --class CalculatorCloneFactory : virtual public CalculatorIfFactory { -- public: -- virtual ~CalculatorCloneFactory() {} -- virtual CalculatorIf* getHandler(const ::apache::thrift::TConnectionInfo& connInfo) -- { -- boost::shared_ptr sock = boost::dynamic_pointer_cast(connInfo.transport); -- cout << "Incoming connection\n"; -- cout << "\tSocketInfo: " << sock->getSocketInfo() << "\n"; -- cout << "\tPeerHost: " << sock->getPeerHost() << "\n"; -- cout << "\tPeerAddress: " << sock->getPeerAddress() << "\n"; -- cout << "\tPeerPort: " << sock->getPeerPort() << "\n"; -- return new CalculatorHandler; -- } -- virtual void releaseHandler( ::shared::SharedServiceIf* handler) { -- delete handler; -- } --}; ++ } ++}; ++ +void RunServer() { + std::string server_address("0.0.0.0:50051"); + GreeterServiceImpl service; @@ -1227,58 +1334,14 @@ index eafffa9..c838b61 100644 + // responsible for shutting down the server for this call to ever return. + server->Wait(); +} - - int main() { -- TThreadedServer server( -- boost::make_shared(boost::make_shared()), -- boost::make_shared(9090), //port -- boost::make_shared(), -- boost::make_shared()); -- -- /* -- // if you don't need per-connection state, do the following instead -- TThreadedServer server( -- boost::make_shared(boost::make_shared()), -- boost::make_shared(9090), //port -- boost::make_shared(), -- boost::make_shared()); -- */ -- -- /** -- * Here are some alternate server types... -- -- // This server only allows one connection at a time, but spawns no threads -- TSimpleServer server( -- boost::make_shared(boost::make_shared()), -- boost::make_shared(9090), -- boost::make_shared(), -- boost::make_shared()); -- -- const int workerCount = 4; -- -- boost::shared_ptr threadManager = -- ThreadManager::newSimpleThreadManager(workerCount); -- threadManager->threadFactory( -- boost::make_shared()); -- threadManager->start(); -- -- // This server allows "workerCount" connection at a time, and reuses threads -- TThreadPoolServer server( -- boost::make_shared(boost::make_shared()), -- boost::make_shared(9090), -- boost::make_shared(), -- boost::make_shared(), -- threadManager); -- */ ++ ++int main() { + RunServer(); - -- cout << "Starting the server..." << endl; -- server.serve(); -- cout << "Done." << endl; - return 0; - } ++ ++ return 0; ++} diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am -index 184a69d..581f75e 100755 +index 184a69d..39d85e1 100755 --- a/tutorial/cpp/Makefile.am +++ b/tutorial/cpp/Makefile.am @@ -18,44 +18,38 @@ @@ -1325,8 +1388,9 @@ index 184a69d..581f75e 100755 + TestClient -TutorialServer_SOURCES = \ +- CppServer.cpp +TestServer_SOURCES = \ - CppServer.cpp ++ GrpcServer.cpp -TutorialServer_LDADD = \ - libtutorialgencpp.la \ @@ -1335,8 +1399,9 @@ index 184a69d..581f75e 100755 $(top_builddir)/lib/cpp/libthrift.la -TutorialClient_SOURCES = \ +- CppClient.cpp +TestClient_SOURCES = \ - CppClient.cpp ++ GrpcClient.cpp -TutorialClient_LDADD = \ - libtutorialgencpp.la \ @@ -1345,7 +1410,7 @@ index 184a69d..581f75e 100755 $(top_builddir)/lib/cpp/libthrift.la # -@@ -63,21 +57,21 @@ TutorialClient_LDADD = \ +@@ -63,26 +57,26 @@ TutorialClient_LDADD = \ # THRIFT = $(top_builddir)/compiler/cpp/thrift @@ -1374,6 +1439,13 @@ index 184a69d..581f75e 100755 style-local: $(CPPSTYLE_CMD) + + EXTRA_DIST = \ + CMakeLists.txt \ +- CppClient.cpp \ +- CppServer.cpp ++ GrpcClient.cpp \ ++ GrpcServer.cpp diff --git a/tutorial/cpp/test.thrift b/tutorial/cpp/test.thrift new file mode 100644 index 0000000..de3c9a4 @@ -1398,10 +1470,10 @@ index 0000000..de3c9a4 2.8.0.rc3.226.g39d4020 -From 1d47ed062e62d136c3db9f6fc1dde9e2f794cf22 Mon Sep 17 00:00:00 2001 +From 096042c132126536870eea118127cf1e608969bc Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 -Subject: [PATCH 3/5] grpc java plugins generator +Subject: [PATCH 3/3] grpc java plugins generator for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thrift --- @@ -1410,7 +1482,7 @@ for examples refer to https://github.com/grpc/grpc-java/tree/master/examples/thr 2 files changed, 887 insertions(+), 27 deletions(-) diff --git a/compiler/cpp/src/generate/t_java_generator.cc b/compiler/cpp/src/generate/t_java_generator.cc -index 2db8cb8..95e1ca8 100644 +index 2db8cb8..8b28fe2 100644 --- a/compiler/cpp/src/generate/t_java_generator.cc +++ b/compiler/cpp/src/generate/t_java_generator.cc @@ -97,10 +97,10 @@ public: @@ -1487,7 +1559,7 @@ index 2db8cb8..95e1ca8 100644 +} + +string t_java_generator::import_extended_service(t_service* tservice) { -+ if (tservice == nullptr) { ++ if (!tservice) { + return string() + "\n"; + } + string ns = tservice->get_program()->get_namespace("java"); @@ -1575,7 +1647,7 @@ index 2db8cb8..95e1ca8 100644 + f_service_ << indent() << + "@java.lang.Deprecated public static interface " << service_name_; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + f_service_ << " extends " << tservice->get_extends()->get_name() + "Grpc." << + tservice->get_extends()->get_name() << endl; + } @@ -1610,7 +1682,7 @@ index 2db8cb8..95e1ca8 100644 + } + f_service_ << endl; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + f_service_ << indent() << "// ARG IDs for extended service" << endl; + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); @@ -1663,7 +1735,7 @@ index 2db8cb8..95e1ca8 100644 + endl << indent() << " break;" << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -1721,7 +1793,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc" ; @@ -1786,7 +1858,7 @@ index 2db8cb8..95e1ca8 100644 + indent_down(); + } + -+ if(tservice->get_extends() != nullptr) { ++ if(tservice->get_extends()) { + t_service* extends_service = tservice->get_extends(); + functions = extends_service->get_functions(); + string extend_service_name = extends_service->get_name() + "Grpc"; @@ -1886,7 +1958,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -1930,7 +2002,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "@java.lang.Deprecated public static interface " << service_name_ << + "BlockingClient " ; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + string extend_service_name = tservice->get_extends()->get_name(); + f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << + extend_service_name << "BlockingClient " ; @@ -2008,7 +2080,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2050,7 +2122,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "@java.lang.Deprecated public static interface " << service_name_ << + "FutureClient " ; + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + string extend_service_name = tservice->get_extends()->get_name(); + f_service_ << endl << indent() << " extends " << extend_service_name << "Grpc." << + extend_service_name << "FutureClient " ; @@ -2068,7 +2140,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << " " << (*f_iter)->get_name() << "_args request);" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2141,7 +2213,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "}" << endl << endl; + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2176,7 +2248,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "private static final int METHODID_" << + (*f_iter)->get_name() << " = " << i << ";" << endl; + } -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2238,7 +2310,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "break;" << endl << endl; + indent_down(); + } -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2303,7 +2375,7 @@ index 2db8cb8..95e1ca8 100644 + indent() << "," << endl << + indent() << " METHOD_" << (*f_iter)->get_name(); + } -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + for(f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { @@ -2352,7 +2424,7 @@ index 2db8cb8..95e1ca8 100644 + indent_down(); + } + -+ if (tservice->get_extends() != nullptr) { ++ if (tservice->get_extends()) { + t_service* extend_service = tservice->get_extends(); + functions = extend_service->get_functions(); + string extend_service_name = extend_service->get_name() + "Grpc"; @@ -2415,193 +2487,3 @@ index 5865c54..1cffbe6 100755 -- 2.8.0.rc3.226.g39d4020 - -From a9769a0fa08f553da76215ca59a8fd797b92a853 Mon Sep 17 00:00:00 2001 -From: chedeti -Date: Mon, 1 Aug 2016 16:56:36 -0700 -Subject: [PATCH 4/5] maintain consistency with grpc - ---- - compiler/cpp/src/generate/t_cpp_generator.cc | 20 ++++++++++---------- - tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} | 0 - tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} | 0 - tutorial/cpp/Makefile.am | 8 ++++---- - 4 files changed, 14 insertions(+), 14 deletions(-) - rename tutorial/cpp/{CppClient.cpp => GrpcClient.cpp} (100%) - rename tutorial/cpp/{CppServer.cpp => GrpcServer.cpp} (100%) - -diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc -index 9c3399b..4e00129 100644 ---- a/compiler/cpp/src/generate/t_cpp_generator.cc -+++ b/compiler/cpp/src/generate/t_cpp_generator.cc -@@ -1641,7 +1641,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { - << endl; - - t_service* extends_service = tservice->get_extends(); -- if (extends_service != nullptr) { -+ if (extends_service) { - f_header_ << "#include \"" << get_include_prefix(*(extends_service->get_program())) - << extends_service->get_name() << ".grpc.thrift.h\"" << endl; - } -@@ -1733,7 +1733,7 @@ void t_cpp_generator::generate_service(t_service* tservice) { - indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - vector functions = extends_service->get_functions(); - vector::iterator f_iter; - -@@ -1749,9 +1749,9 @@ void t_cpp_generator::generate_service(t_service* tservice) { - "};" << endl; - - // Generate service class -- if ( extends_service != nullptr ) { -+ if ( extends_service) { - f_header_ << "class " << service_name_ << " : public " << -- extends_service->get_name() << " {" << endl << -+ type_name(extends_service) << " {" << endl << - "public:" << endl; - } - else { -@@ -1841,7 +1841,7 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { - void t_cpp_generator::generate_service_stub_interface(t_service* tservice) { - - string extends = ""; -- if (tservice->get_extends() != nullptr) { -+ if (tservice->get_extends()) { - extends = " : virtual public " + type_name(tservice->get_extends()) + "::StubInterface"; - } - -@@ -1890,7 +1890,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - } - - t_service* extends_service = tservice->get_extends(); -- if (extends_service != nullptr) { -+ if (extends_service) { - // generate inherited methods - vector functions = extends_service->get_functions(); - vector::iterator f_iter; -@@ -1914,7 +1914,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - // generate inherited methods - vector functions = extends_service->get_functions(); - vector::iterator f_iter; -@@ -1944,7 +1944,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - // generate inherited methods - vector functions = extends_service->get_functions(); - vector::iterator f_iter; -@@ -2002,7 +2002,7 @@ void t_cpp_generator::generate_service_stub(t_service* tservice) { - - } - -- if (extends_service != nullptr) { -+ if (extends_service) { - vector functions = extends_service->get_functions(); - vector::iterator f_iter; - for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { -@@ -2049,7 +2049,7 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty - } - - string extends = ""; -- if (tservice->get_extends() != NULL) { -+ if (tservice->get_extends()) { - extends = " : virtual public " + type_name(tservice->get_extends()) + style + "::Service"; - if (style == "CobCl" && gen_templates_) { - // TODO(simpkins): If gen_templates_ is enabled, we currently assume all -diff --git a/tutorial/cpp/CppClient.cpp b/tutorial/cpp/GrpcClient.cpp -similarity index 100% -rename from tutorial/cpp/CppClient.cpp -rename to tutorial/cpp/GrpcClient.cpp -diff --git a/tutorial/cpp/CppServer.cpp b/tutorial/cpp/GrpcServer.cpp -similarity index 100% -rename from tutorial/cpp/CppServer.cpp -rename to tutorial/cpp/GrpcServer.cpp -diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am -index 581f75e..39d85e1 100755 ---- a/tutorial/cpp/Makefile.am -+++ b/tutorial/cpp/Makefile.am -@@ -39,14 +39,14 @@ noinst_PROGRAMS = \ - TestClient - - TestServer_SOURCES = \ -- CppServer.cpp -+ GrpcServer.cpp - - TestServer_LDADD = \ - libtestgencpp.la \ - $(top_builddir)/lib/cpp/libthrift.la - - TestClient_SOURCES = \ -- CppClient.cpp -+ GrpcClient.cpp - - TestClient_LDADD = \ - libtestgencpp.la \ -@@ -78,5 +78,5 @@ style-local: - - EXTRA_DIST = \ - CMakeLists.txt \ -- CppClient.cpp \ -- CppServer.cpp -+ GrpcClient.cpp \ -+ GrpcServer.cpp --- -2.8.0.rc3.226.g39d4020 - - -From bc74fca1ee73333819724f51d5aaff3546443ed0 Mon Sep 17 00:00:00 2001 -From: chedeti -Date: Mon, 1 Aug 2016 17:00:17 -0700 -Subject: [PATCH 5/5] fix typo - ---- - tutorial/cpp/GrpcClient.cpp | 4 ++-- - tutorial/cpp/GrpcServer.cpp | 2 +- - 2 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp -index c41604e..ab1fe77 100644 ---- a/tutorial/cpp/GrpcClient.cpp -+++ b/tutorial/cpp/GrpcClient.cpp -@@ -1,6 +1,6 @@ - /* - * -- * Copyright 2015, Google Inc. -+ * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without -@@ -50,7 +50,7 @@ class GreeterClient { - GreeterClient(std::shared_ptr channel) - : stub_(Greeter::NewStub(channel)) {} - -- // Assambles the client's payload, sends it and presents the response back -+ // Assembles the client's payload, sends it and presents the response back - // from the server. - std::string SayHello(const std::string& user) { - // Data we are sending to the server. -diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp -index c838b61..f63db57 100644 ---- a/tutorial/cpp/GrpcServer.cpp -+++ b/tutorial/cpp/GrpcServer.cpp -@@ -1,6 +1,6 @@ - /* - * -- * Copyright 2015, Google Inc. -+ * Copyright 2016, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without --- -2.8.0.rc3.226.g39d4020 - From 82afcaa009e85cba16422b94a23387a3b7a0bd06 Mon Sep 17 00:00:00 2001 From: chedeti Date: Wed, 3 Aug 2016 16:38:05 -0700 Subject: [PATCH 17/43] rename class variables to snake_case --- .../grpc++/impl/codegen/thrift_serializer.h | 94 ++++++++++--------- include/grpc++/impl/codegen/thrift_utils.h | 21 ++--- tools/grift/Dockerfile | 4 +- tools/grift/README.md | 2 +- tools/grift/grpc_plugins_generator.patch | 91 +++++++++--------- 5 files changed, 110 insertions(+), 102 deletions(-) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index 46112ee5b23..04dcc699deb 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -31,12 +31,9 @@ * */ - #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H - #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H +#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H +#define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H -#include -#include -#include #include #include #include @@ -46,6 +43,10 @@ #include #include #include +#include +#include +#include +#include namespace apache { namespace thrift { @@ -59,20 +60,22 @@ using apache::thrift::transport::TMemoryBuffer; using apache::thrift::transport::TBufferBase; using apache::thrift::transport::TTransport; -template class ThriftSerializer { -public: +template +class ThriftSerializer { + public: ThriftSerializer() - : prepared_ (false) - , last_deserialized_ (false) - , serialize_version_ (false) {} + : prepared_(false), + last_deserialized_(false), + serialize_version_(false) {} virtual ~ThriftSerializer() {} // Serialize the passed type into the internal buffer // and returns a pointer to internal buffer and its size - template void Serialize(const T& fields, const uint8_t** serializedBuffer, - size_t* serializedLen) { - // prepare or reset buffer + template + void Serialize(const T& fields, const uint8_t** serialized_buffer, + size_t* serialized_len) { + // prepare or reset buffer if (!prepared_ || last_deserialized_) { prepare(); } else { @@ -85,7 +88,7 @@ public: protocol_->writeMessageBegin("", TMessageType(0), 0); } - // serilaize fields into buffer + // serialize fields into buffer fields.write(protocol_.get()); // write the end of message @@ -93,22 +96,23 @@ public: protocol_->writeMessageEnd(); } - uint8_t* byteBuffer; - uint32_t byteBufferSize; - buffer_->getBuffer(&byteBuffer, &byteBufferSize); - *serializedBuffer = byteBuffer; - *serializedLen = byteBufferSize; + uint8_t* byte_buffer; + uint32_t byte_buffer_size; + buffer_->getBuffer(&byte_buffer, &byte_buffer_size); + *serialized_buffer = byte_buffer; + *serialized_len = byte_buffer_size; } // Serialize the passed type into the byte buffer - template void Serialize(const T& fields, grpc_byte_buffer** bp) { + template + void Serialize(const T& fields, grpc_byte_buffer** bp) { + const uint8_t* byte_buffer; + size_t byte_buffer_size; - const uint8_t* byteBuffer; - size_t byteBufferSize; + Serialize(fields, &byte_buffer, &byte_buffer_size); - Serialize(fields, &byteBuffer, &byteBufferSize); - - gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + gpr_slice slice = + gpr_slice_from_copied_buffer((char*)byte_buffer, byte_buffer_size); *bp = grpc_raw_byte_buffer_create(&slice, 1); @@ -117,21 +121,22 @@ public: // Deserialize the passed char array into the passed type, returns the number // of bytes that have been consumed from the passed string. - template uint32_t Deserialize(const uint8_t* serializedBuffer, size_t length, - T* fields) { + template + uint32_t Deserialize(const uint8_t* serialized_buffer, size_t length, + T* fields) { // prepare buffer if necessary if (!prepared_) { prepare(); } last_deserialized_ = true; - //reset buffer transport - buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + // reset buffer transport + buffer_->resetBuffer((uint8_t*)serialized_buffer, length); // read the protocol version if necessary if (serialize_version_) { std::string name = ""; - TMessageType mt = (TMessageType) 0; + TMessageType mt = (TMessageType)0; int32_t seq_id = 0; protocol_->readMessageBegin(name, mt, seq_id); } @@ -147,17 +152,17 @@ public: return len; } - // Deserialize the passed byte buffer to passed type, returns the number // of bytes consumed from byte buffer - template uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { - + template + uint32_t Deserialize(grpc_byte_buffer* buffer, T* msg) { grpc_byte_buffer_reader reader; grpc_byte_buffer_reader_init(&reader, buffer); gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); - uint32_t len = Deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), msg); + uint32_t len = + Deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), msg); gpr_slice_unref(slice); @@ -167,9 +172,7 @@ public: } // set serialization version flag - void SetSerializeVersion(bool value) { - serialize_version_ = value; - } + void SetSerializeVersion(bool value) { serialize_version_ = value; } // Set the container size limit to deserialize // This function should be called after buffer_ is initialized @@ -189,7 +192,7 @@ public: protocol_->setStringSizeLimit(string_limit); } -private: + private: bool prepared_; bool last_deserialized_; boost::shared_ptr buffer_; @@ -197,6 +200,7 @@ private: bool serialize_version_; void prepare() { + buffer_.reset(new TMemoryBuffer()); // create a protocol for the memory buffer transport @@ -205,13 +209,15 @@ private: prepared_ = true; } -}; // ThriftSerializer +}; // ThriftSerializer -typedef ThriftSerializer> ThriftSerializerBinary; -typedef ThriftSerializer> ThriftSerializerCompact; +typedef ThriftSerializer> + ThriftSerializerBinary; +typedef ThriftSerializer> + ThriftSerializerCompact; -} // namespace util -} // namespace thrift -} // namespace apache +} // namespace util +} // namespace thrift +} // namespace apache #endif \ No newline at end of file diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h index 14332c05219..7d19b247f4c 100644 --- a/include/grpc++/impl/codegen/thrift_utils.h +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -34,16 +34,16 @@ #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H #define GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H -#include -#include -#include -#include #include #include #include #include #include #include +#include +#include +#include +#include #include #include @@ -52,23 +52,20 @@ namespace grpc { using apache::thrift::util::ThriftSerializerCompact; template -class SerializationTraits::value>::type> { +class SerializationTraits::value>::type> { public: - - static Status Serialize(const T& msg, - grpc_byte_buffer** bp, bool* own_buffer) { - + static Status Serialize(const T& msg, grpc_byte_buffer** bp, + bool* own_buffer) { *own_buffer = true; ThriftSerializerCompact serializer; - serializer.Serialize(msg, bp); return Status(StatusCode::OK, "ok"); } - static Status Deserialize(grpc_byte_buffer* buffer, - T* msg, + static Status Deserialize(grpc_byte_buffer* buffer, T* msg, int max_message_size) { if (!buffer) { return Status(StatusCode::INTERNAL, "No payload"); diff --git a/tools/grift/Dockerfile b/tools/grift/Dockerfile index 5238010ea92..954640f0df0 100644 --- a/tools/grift/Dockerfile +++ b/tools/grift/Dockerfile @@ -46,8 +46,8 @@ RUN apt-get update && \ curl make automake libtool # Configure git -RUN git config --global user.name " " && \ - git config --global user.email " " +RUN git config --global user.name "Jenkins" && \ + git config --global user.email "jenkins@grpc" RUN git clone https://github.com/grpc/grpc diff --git a/tools/grift/README.md b/tools/grift/README.md index 25c67451326..2525f9b83dd 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -2,7 +2,7 @@ Copyright 2016 Google Inc. #Documentation -grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with GRPC. +grift is integration of [Apache Thrift](https://github.com/apache/thrift.git) Serializer with gRPC. This integration allows you to use grpc to send thrift messages in C++ and java. diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index 3a6710c224e..a1d4cecde04 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -59,7 +59,7 @@ index 6fd15d2..7de1fad 100755 2.8.0.rc3.226.g39d4020 -From c8577ad5513543c57a81ad1bf4927cc8a78baa03 Mon Sep 17 00:00:00 2001 +From e724d3abf096278615085bd58217321e32b43fd8 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 Subject: [PATCH 2/3] grpc cpp plugins generator with example @@ -69,16 +69,16 @@ Subject: [PATCH 2/3] grpc cpp plugins generator with example tutorial/cpp/CMakeLists.txt | 53 --- tutorial/cpp/CppClient.cpp | 80 ----- tutorial/cpp/CppServer.cpp | 181 ---------- - tutorial/cpp/GrpcClient.cpp | 94 ++++++ - tutorial/cpp/GrpcServer.cpp | 87 +++++ + tutorial/cpp/GriftClient.cpp | 93 ++++++ + tutorial/cpp/GriftServer.cpp | 93 ++++++ tutorial/cpp/Makefile.am | 66 ++-- tutorial/cpp/test.thrift | 13 + - 8 files changed, 636 insertions(+), 416 deletions(-) + 8 files changed, 641 insertions(+), 416 deletions(-) delete mode 100644 tutorial/cpp/CMakeLists.txt delete mode 100644 tutorial/cpp/CppClient.cpp delete mode 100644 tutorial/cpp/CppServer.cpp - create mode 100644 tutorial/cpp/GrpcClient.cpp - create mode 100644 tutorial/cpp/GrpcServer.cpp + create mode 100644 tutorial/cpp/GriftClient.cpp + create mode 100644 tutorial/cpp/GriftServer.cpp create mode 100644 tutorial/cpp/test.thrift diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -1147,12 +1147,12 @@ index eafffa9..0000000 - cout << "Done." << endl; - return 0; -} -diff --git a/tutorial/cpp/GrpcClient.cpp b/tutorial/cpp/GrpcClient.cpp +diff --git a/tutorial/cpp/GriftClient.cpp b/tutorial/cpp/GriftClient.cpp new file mode 100644 -index 0000000..ab1fe77 +index 0000000..647a683 --- /dev/null -+++ b/tutorial/cpp/GrpcClient.cpp -@@ -0,0 +1,94 @@ ++++ b/tutorial/cpp/GriftClient.cpp +@@ -0,0 +1,93 @@ +/* + * + * Copyright 2016, Google Inc. @@ -1198,7 +1198,6 @@ index 0000000..ab1fe77 +using grpc::ClientContext; +using grpc::Status; +using test::Greeter; -+using namespace test; + +class GreeterClient { + public: @@ -1247,12 +1246,12 @@ index 0000000..ab1fe77 + + return 0; +} -diff --git a/tutorial/cpp/GrpcServer.cpp b/tutorial/cpp/GrpcServer.cpp +diff --git a/tutorial/cpp/GriftServer.cpp b/tutorial/cpp/GriftServer.cpp new file mode 100644 -index 0000000..f63db57 +index 0000000..7c01606 --- /dev/null -+++ b/tutorial/cpp/GrpcServer.cpp -@@ -0,0 +1,87 @@ ++++ b/tutorial/cpp/GriftServer.cpp +@@ -0,0 +1,93 @@ +/* + * + * Copyright 2016, Google Inc. @@ -1301,11 +1300,14 @@ index 0000000..f63db57 +using grpc::Status; +using test::Greeter; + -+using namespace grpc; -+using namespace test; -+ +// Logic and data behind the server's behavior. +class GreeterServiceImpl final : public Greeter::Service { ++ public: ++ ~GreeterServiceImpl() { ++ // shutdown server ++ server->Shutdown(); ++ } ++ + Status SayHello(ServerContext* context,const Greeter::SayHelloReq* request, + Greeter::SayHelloResp* reply) override { + std::string prefix("Hello "); @@ -1314,34 +1316,37 @@ index 0000000..f63db57 + + return Status::OK; + } -+}; + -+void RunServer() { -+ std::string server_address("0.0.0.0:50051"); -+ GreeterServiceImpl service; ++ void RunServer() { ++ std::string server_address("0.0.0.0:50051"); ++ ++ ServerBuilder builder; ++ // Listen on the given address without any authentication mechanism. ++ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); ++ // Register "service" as the instance through which we'll communicate with ++ // clients. In this case it corresponds to an *synchronous* service. ++ builder.RegisterService(this); ++ // Finally assemble the server. ++ server = builder.BuildAndStart(); ++ std::cout << "Server listening on " << server_address << std::endl; ++ ++ // Wait for the server to shutdown. Note that some other thread must be ++ // responsible for shutting down the server for this call to ever return. ++ server->Wait(); ++ } + -+ ServerBuilder builder; -+ // Listen on the given address without any authentication mechanism. -+ builder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); -+ // Register "service" as the instance through which we'll communicate with -+ // clients. In this case it corresponds to an *synchronous* service. -+ builder.RegisterService(&service); -+ // Finally assemble the server. -+ std::unique_ptr server(builder.BuildAndStart()); -+ std::cout << "Server listening on " << server_address << std::endl; -+ -+ // Wait for the server to shutdown. Note that some other thread must be -+ // responsible for shutting down the server for this call to ever return. -+ server->Wait(); -+} ++ private: ++ std::unique_ptr server; ++}; + +int main() { -+ RunServer(); ++ GreeterServiceImpl service; ++ service.RunServer(); + + return 0; +} diff --git a/tutorial/cpp/Makefile.am b/tutorial/cpp/Makefile.am -index 184a69d..39d85e1 100755 +index 184a69d..6f91e28 100755 --- a/tutorial/cpp/Makefile.am +++ b/tutorial/cpp/Makefile.am @@ -18,44 +18,38 @@ @@ -1390,7 +1395,7 @@ index 184a69d..39d85e1 100755 -TutorialServer_SOURCES = \ - CppServer.cpp +TestServer_SOURCES = \ -+ GrpcServer.cpp ++ GriftServer.cpp -TutorialServer_LDADD = \ - libtutorialgencpp.la \ @@ -1401,7 +1406,7 @@ index 184a69d..39d85e1 100755 -TutorialClient_SOURCES = \ - CppClient.cpp +TestClient_SOURCES = \ -+ GrpcClient.cpp ++ GriftClient.cpp -TutorialClient_LDADD = \ - libtutorialgencpp.la \ @@ -1444,8 +1449,8 @@ index 184a69d..39d85e1 100755 CMakeLists.txt \ - CppClient.cpp \ - CppServer.cpp -+ GrpcClient.cpp \ -+ GrpcServer.cpp ++ GriftClient.cpp \ ++ GriftServer.cpp diff --git a/tutorial/cpp/test.thrift b/tutorial/cpp/test.thrift new file mode 100644 index 0000000..de3c9a4 @@ -1470,7 +1475,7 @@ index 0000000..de3c9a4 2.8.0.rc3.226.g39d4020 -From 096042c132126536870eea118127cf1e608969bc Mon Sep 17 00:00:00 2001 +From f991f33dd6461eae197b6ad0e7088b571f2a7b22 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 Subject: [PATCH 3/3] grpc java plugins generator From 2e698457f370091111d0fe620bad306c743a3548 Mon Sep 17 00:00:00 2001 From: chedeti Date: Wed, 3 Aug 2016 16:45:18 -0700 Subject: [PATCH 18/43] sanity check for thrift --- tools/run_tests/sanity/check_submodules.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index b602d695649..e2bbd8cabfc 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -47,6 +47,7 @@ cat << EOF | awk '{ print $1 }' | sort > $want_submodules f8ac463766281625ad710900479130c7fcb4d63b third_party/nanopb (nanopb-0.3.4-29-gf8ac463) bdeb215cab2985195325fcd5e70c3fa751f46e0f third_party/protobuf (v3.0.0-beta-3.3) 50893291621658f355bc5b4d450a8d06a563053d third_party/zlib (v1.2.8) + bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c third_party/thrift EOF diff -u $submodules $want_submodules From 76db7f9a7cdf6b5fd6706037c72de1dd131b181e Mon Sep 17 00:00:00 2001 From: chedeti Date: Thu, 4 Aug 2016 11:56:54 -0700 Subject: [PATCH 19/43] use boost::make_shared --- include/grpc++/impl/codegen/thrift_serializer.h | 14 ++++++-------- tools/grift/README.md | 3 +-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index 04dcc699deb..fcb0ffaad6e 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -111,8 +111,8 @@ class ThriftSerializer { Serialize(fields, &byte_buffer, &byte_buffer_size); - gpr_slice slice = - gpr_slice_from_copied_buffer((char*)byte_buffer, byte_buffer_size); + gpr_slice slice = gpr_slice_from_copied_buffer( + reinterpret_cast(byte_buffer), byte_buffer_size); *bp = grpc_raw_byte_buffer_create(&slice, 1); @@ -131,12 +131,12 @@ class ThriftSerializer { last_deserialized_ = true; // reset buffer transport - buffer_->resetBuffer((uint8_t*)serialized_buffer, length); + buffer_->resetBuffer(const_cast(serialized_buffer), length); // read the protocol version if necessary if (serialize_version_) { std::string name = ""; - TMessageType mt = (TMessageType)0; + TMessageType mt = static_cast(0); int32_t seq_id = 0; protocol_->readMessageBegin(name, mt, seq_id); } @@ -200,11 +200,9 @@ class ThriftSerializer { bool serialize_version_; void prepare() { - - buffer_.reset(new TMemoryBuffer()); - + buffer_ = boost::make_shared(*(new TMemoryBuffer())); // create a protocol for the memory buffer transport - protocol_.reset(new Protocol(buffer_)); + protocol_ = std::make_shared(*(new Protocol(buffer_))); prepared_ = true; } diff --git a/tools/grift/README.md b/tools/grift/README.md index 2525f9b83dd..7cbbdc567bf 100644 --- a/tools/grift/README.md +++ b/tools/grift/README.md @@ -23,5 +23,4 @@ grift uses Compact Protocol to serialize thrift messages. #Installation Before Installing thrift make sure to apply this [patch](grpc_plugins_generator.patch) to third_party/thrift. -Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to -install thrift. \ No newline at end of file +Go to third_party/thrift and follow the [INSTALLATION](https://github.com/apache/thrift.git) instructions to install thrift with commit id bcad91771b7f0bff28a1cac1981d7ef2b9bcef3c. \ No newline at end of file From a7ee93864a2e822ec510dccec19012e59acdeb7d Mon Sep 17 00:00:00 2001 From: chedeti Date: Thu, 4 Aug 2016 14:42:35 -0700 Subject: [PATCH 20/43] remove const in Deserialize --- include/grpc++/impl/codegen/thrift_serializer.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h index fcb0ffaad6e..7308a1577c8 100644 --- a/include/grpc++/impl/codegen/thrift_serializer.h +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -122,8 +122,7 @@ class ThriftSerializer { // Deserialize the passed char array into the passed type, returns the number // of bytes that have been consumed from the passed string. template - uint32_t Deserialize(const uint8_t* serialized_buffer, size_t length, - T* fields) { + uint32_t Deserialize(uint8_t* serialized_buffer, size_t length, T* fields) { // prepare buffer if necessary if (!prepared_) { prepare(); @@ -131,7 +130,7 @@ class ThriftSerializer { last_deserialized_ = true; // reset buffer transport - buffer_->resetBuffer(const_cast(serialized_buffer), length); + buffer_->resetBuffer(serialized_buffer, length); // read the protocol version if necessary if (serialize_version_) { @@ -200,10 +199,9 @@ class ThriftSerializer { bool serialize_version_; void prepare() { - buffer_ = boost::make_shared(*(new TMemoryBuffer())); + buffer_ = boost::make_shared(); // create a protocol for the memory buffer transport - protocol_ = std::make_shared(*(new Protocol(buffer_))); - + protocol_ = std::make_shared(buffer_); prepared_ = true; } From d07c17e3430bbf1cc1a802b76cf57175bbb6603c Mon Sep 17 00:00:00 2001 From: chedeti Date: Thu, 4 Aug 2016 17:52:00 -0700 Subject: [PATCH 21/43] fix Dockerfile --- tools/grift/Dockerfile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/grift/Dockerfile b/tools/grift/Dockerfile index 954640f0df0..223ee939927 100644 --- a/tools/grift/Dockerfile +++ b/tools/grift/Dockerfile @@ -43,21 +43,25 @@ RUN apt-get update && \ cmake \ libiberty-dev \ g++ unzip \ - curl make automake libtool + curl make automake libtool libboost-dev # Configure git RUN git config --global user.name "Jenkins" && \ git config --global user.email "jenkins@grpc" +# Clone gRPC RUN git clone https://github.com/grpc/grpc +# Update Submodules RUN cd grpc && git submodule update --init -RUN cd grpc/third_party/thrift && git am --signoff < ../../tools/grift/grpc_plugins_generator.patch - +# Install protobuf RUN cd grpc/third_party/protobuf && ./autogen.sh && ./configure && \ make -j && make check -j && make install && ldconfig +# Install gRPC RUN cd grpc && make -j && make install -RUN cd grpc/third_party/thrift && ./bootstrap.sh && ./configure && make -j && make install \ No newline at end of file +# Install thrift +RUN cd grpc/third_party/thrift && git am --signoff < ../../tools/grift/grpc_plugins_generator.patch && \ + ./bootstrap.sh && ./configure && make -j && make install \ No newline at end of file From 7e024be839687470bd1343f70f08ec1e703eb9ec Mon Sep 17 00:00:00 2001 From: chedeti Date: Fri, 5 Aug 2016 11:15:37 -0700 Subject: [PATCH 22/43] fix multilevel inheritence codegen --- tools/grift/grpc_plugins_generator.patch | 75 ++++++++++++++---------- 1 file changed, 43 insertions(+), 32 deletions(-) diff --git a/tools/grift/grpc_plugins_generator.patch b/tools/grift/grpc_plugins_generator.patch index a1d4cecde04..de82a01f625 100644 --- a/tools/grift/grpc_plugins_generator.patch +++ b/tools/grift/grpc_plugins_generator.patch @@ -59,21 +59,21 @@ index 6fd15d2..7de1fad 100755 2.8.0.rc3.226.g39d4020 -From e724d3abf096278615085bd58217321e32b43fd8 Mon Sep 17 00:00:00 2001 +From 387e4300bc9d98176a92a7c010621443a538e7f2 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:16:40 -0700 Subject: [PATCH 2/3] grpc cpp plugins generator with example --- - compiler/cpp/src/generate/t_cpp_generator.cc | 478 +++++++++++++++++++++++---- + compiler/cpp/src/generate/t_cpp_generator.cc | 489 +++++++++++++++++++++++---- tutorial/cpp/CMakeLists.txt | 53 --- tutorial/cpp/CppClient.cpp | 80 ----- tutorial/cpp/CppServer.cpp | 181 ---------- - tutorial/cpp/GriftClient.cpp | 93 ++++++ - tutorial/cpp/GriftServer.cpp | 93 ++++++ + tutorial/cpp/GriftClient.cpp | 93 +++++ + tutorial/cpp/GriftServer.cpp | 93 +++++ tutorial/cpp/Makefile.am | 66 ++-- tutorial/cpp/test.thrift | 13 + - 8 files changed, 641 insertions(+), 416 deletions(-) + 8 files changed, 652 insertions(+), 416 deletions(-) delete mode 100644 tutorial/cpp/CMakeLists.txt delete mode 100644 tutorial/cpp/CppClient.cpp delete mode 100644 tutorial/cpp/CppServer.cpp @@ -82,7 +82,7 @@ Subject: [PATCH 2/3] grpc cpp plugins generator with example create mode 100644 tutorial/cpp/test.thrift diff --git a/compiler/cpp/src/generate/t_cpp_generator.cc b/compiler/cpp/src/generate/t_cpp_generator.cc -index 6c04899..4e00129 100644 +index 6c04899..1557241 100644 --- a/compiler/cpp/src/generate/t_cpp_generator.cc +++ b/compiler/cpp/src/generate/t_cpp_generator.cc @@ -162,6 +162,8 @@ public: @@ -328,7 +328,7 @@ index 6c04899..4e00129 100644 << endl; f_service_tcc_ << "#ifndef " << svcname << "_TCC" << endl << "#define " << svcname << "_TCC" -@@ -1663,19 +1704,66 @@ void t_cpp_generator::generate_service(t_service* tservice) { +@@ -1663,19 +1704,69 @@ void t_cpp_generator::generate_service(t_service* tservice) { } } @@ -361,15 +361,18 @@ index 6c04899..4e00129 100644 + indent() << "\"/" << ns << "." << service_name_ << "/" << (*f_iter)->get_name() << "\"," << endl; + } + -+ if (extends_service) { -+ vector functions = extends_service->get_functions(); ++ ++ t_service* service_iter = extends_service; ++ while (service_iter) { ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + + for ( f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_service_ << -+ indent() << "\"/" << extends_service->get_program()->get_namespace("cpp") << -+ "." << extends_service->get_name() << "/" << (*f_iter)->get_name() << "\"," << endl; ++ indent() << "\"/" << service_iter->get_program()->get_namespace("cpp") << ++ "." << service_iter->get_name() << "/" << (*f_iter)->get_name() << "\"," << endl; + } ++ service_iter = service_iter->get_extends(); + } + + indent_down(); @@ -403,7 +406,7 @@ index 6c04899..4e00129 100644 // Generate all the cob components if (gen_cob_style_) { -@@ -1688,10 +1776,14 @@ void t_cpp_generator::generate_service(t_service* tservice) { +@@ -1688,10 +1779,14 @@ void t_cpp_generator::generate_service(t_service* tservice) { generate_service_async_skeleton(tservice); } @@ -418,7 +421,7 @@ index 6c04899..4e00129 100644 // Close the namespace f_service_ << ns_close_ << endl << endl; f_service_tcc_ << ns_close_ << endl << endl; -@@ -1729,15 +1821,11 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { +@@ -1729,15 +1824,11 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { string name_orig = ts->get_name(); // TODO(dreiss): Why is this stuff not in generate_function_helpers? @@ -436,7 +439,7 @@ index 6c04899..4e00129 100644 ts->set_name(name_orig); generate_function_helpers(tservice, *f_iter); -@@ -1745,13 +1833,210 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { +@@ -1745,13 +1836,218 @@ void t_cpp_generator::generate_service_helpers(t_service* tservice) { } /** @@ -497,9 +500,10 @@ index 6c04899..4e00129 100644 + } + + t_service* extends_service = tservice->get_extends(); -+ if (extends_service) { ++ t_service* service_iter = extends_service; ++ while (service_iter) { + // generate inherited methods -+ vector functions = extends_service->get_functions(); ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string function_name = (*f_iter)->get_name(); @@ -508,6 +512,7 @@ index 6c04899..4e00129 100644 + "(::grpc::ClientContext* context, const " << function_name << + "Req& request, " << function_name << "Resp* response) override;" << endl; + } ++ service_iter = service_iter->get_extends(); + } + + f_header_ << @@ -521,14 +526,16 @@ index 6c04899..4e00129 100644 + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } + -+ if (extends_service) { ++ service_iter = extends_service; ++ while (service_iter) { + // generate inherited methods -+ vector functions = extends_service->get_functions(); ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + f_header_ << + indent() << "const ::grpc::RpcMethod rpcmethod_" << (*f_iter)->get_name() << "_;" << endl; + } ++ service_iter = service_iter->get_extends(); + } + + indent_down(); @@ -551,9 +558,10 @@ index 6c04899..4e00129 100644 + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } + -+ if (extends_service) { ++ service_iter = extends_service; ++ while (service_iter) { + // generate inherited methods -+ vector functions = extends_service->get_functions(); ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter, ++i) { + f_service_ << @@ -561,6 +569,7 @@ index 6c04899..4e00129 100644 + ", rpcmethod_" << (*f_iter)->get_name() << "_(" << + service_name_ << "_method_names[" << i << "], ::grpc::RpcMethod::NORMAL_RPC, channel)" << endl; + } ++ service_iter = service_iter->get_extends(); + } + f_service_ << + indent() << "{}" << endl; @@ -609,8 +618,9 @@ index 6c04899..4e00129 100644 + + } + -+ if (extends_service) { -+ vector functions = extends_service->get_functions(); ++ service_iter = extends_service; ++ while (service_iter) { ++ vector functions = service_iter->get_functions(); + vector::iterator f_iter; + for (f_iter = functions.begin(); f_iter != functions.end(); ++f_iter) { + string function_name = (*f_iter)->get_name(); @@ -631,6 +641,7 @@ index 6c04899..4e00129 100644 + "}" << endl; + + } ++ service_iter = service_iter->get_extends(); + } + +} @@ -648,7 +659,7 @@ index 6c04899..4e00129 100644 if (style == "CobCl") { // Forward declare the client. string client_name = service_name_ + "CobClient"; -@@ -1764,13 +2049,15 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1764,13 +2060,15 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty } string extends = ""; @@ -666,7 +677,7 @@ index 6c04899..4e00129 100644 } if (style == "CobCl" && gen_templates_) { -@@ -1778,7 +2065,9 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1778,7 +2076,9 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty } f_header_ << "class " << service_if_name << extends << " {" << endl << " public:" << endl; indent_up(); @@ -677,7 +688,7 @@ index 6c04899..4e00129 100644 vector functions = tservice->get_functions(); vector::iterator f_iter; -@@ -1786,7 +2075,12 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1786,7 +2086,12 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty if ((*f_iter)->has_doc()) f_header_ << endl; generate_java_doc(f_header_, *f_iter); @@ -691,7 +702,7 @@ index 6c04899..4e00129 100644 } indent_down(); f_header_ << "};" << endl << endl; -@@ -1797,6 +2091,66 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty +@@ -1797,6 +2102,66 @@ void t_cpp_generator::generate_service_interface(t_service* tservice, string sty f_header_ << "typedef " << service_if_name << "< ::apache::thrift::protocol::TProtocol> " << service_name_ << style << "If;" << endl << endl; } @@ -758,7 +769,7 @@ index 6c04899..4e00129 100644 } /** -@@ -3095,7 +3449,7 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* +@@ -3095,7 +3460,7 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* std::ofstream& out = (gen_templates_ ? f_service_tcc_ : f_service_); @@ -767,7 +778,7 @@ index 6c04899..4e00129 100644 t_field success(tfunction->get_returntype(), "success", 0); if (!tfunction->get_returntype()->is_void()) { result.append(&success); -@@ -3109,17 +3463,9 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* +@@ -3109,17 +3474,9 @@ void t_cpp_generator::generate_function_helpers(t_service* tservice, t_function* } generate_struct_declaration(f_header_, &result, false); @@ -786,7 +797,7 @@ index 6c04899..4e00129 100644 } /** -@@ -3162,8 +3508,8 @@ void t_cpp_generator::generate_process_function(t_service* tservice, +@@ -3162,8 +3519,8 @@ void t_cpp_generator::generate_process_function(t_service* tservice, << endl; scope_up(out); @@ -797,7 +808,7 @@ index 6c04899..4e00129 100644 if (tfunction->is_oneway() && !unnamed_oprot_seqid) { out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; -@@ -3320,7 +3666,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, +@@ -3320,7 +3677,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, out << indent() << "(void) seqid;" << endl << indent() << "(void) oprot;" << endl; } @@ -806,7 +817,7 @@ index 6c04899..4e00129 100644 << indent() << "void* ctx = NULL;" << endl << indent() << "if (this->eventHandler_.get() != NULL) {" << endl << indent() << " ctx = this->eventHandler_->getContext(" << service_func_name << ", NULL);" << endl -@@ -3487,7 +3833,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, +@@ -3487,7 +3844,7 @@ void t_cpp_generator::generate_process_function(t_service* tservice, << "this->eventHandler_.get(), ctx, " << service_func_name << ");" << endl << endl; // Throw the TDelayedException, and catch the result @@ -1475,7 +1486,7 @@ index 0000000..de3c9a4 2.8.0.rc3.226.g39d4020 -From f991f33dd6461eae197b6ad0e7088b571f2a7b22 Mon Sep 17 00:00:00 2001 +From 3e4d75a2e2c474ee7700e7c9acaf89fdb768bedc Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 16:23:53 -0700 Subject: [PATCH 3/3] grpc java plugins generator From a6bdb30311383a49337529ebde7976ba9239dab9 Mon Sep 17 00:00:00 2001 From: Stanley Cheung Date: Tue, 9 Aug 2016 08:54:39 -0700 Subject: [PATCH 23/43] fix jenkins linux image in create script --- tools/gce/create_linux_worker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/gce/create_linux_worker.sh b/tools/gce/create_linux_worker.sh index 8a2df40859b..7bf8b240818 100755 --- a/tools/gce/create_linux_worker.sh +++ b/tools/gce/create_linux_worker.sh @@ -43,8 +43,8 @@ gcloud compute instances create $INSTANCE_NAME \ --project="$CLOUD_PROJECT" \ --zone "$ZONE" \ --machine-type n1-standard-8 \ - --image-family=ubuntu-1510 \ - --image-project=ubuntu-os-cloud \ + --image=ubuntu-1510 \ + --image-project=grpc-testing \ --boot-disk-size 1000 echo 'Created GCE instance, waiting 60 seconds for it to come online.' From f07506438c012f1f466670f05284594ca6808a26 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 9 Aug 2016 14:25:08 -0700 Subject: [PATCH 24/43] Work in progress. Do not check in yet. --- .../cronet/transport/cronet_transport.c | 541 ++++++++++++------ 1 file changed, 359 insertions(+), 182 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 694d346fc33..e8746b4e6e6 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -51,6 +51,20 @@ #define GRPC_HEADER_SIZE_IN_BYTES 5 +#define CRONET_LOG(...) {if (1) gpr_log(__VA_ARGS__);} + +enum OP_RESULT { + ACTION_TAKEN_WITH_CALLBACK, + ACTION_TAKEN_NO_CALLBACK, + NO_ACTION_POSSIBLE +}; + +const char *OP_RESULT_STRING[] = { + "ACTION_TAKEN_WITH_CALLBACK", + "ACTION_TAKEN_NO_CALLBACK", + "NO_ACTION_POSSIBLE" +}; + enum OP_ID { OP_SEND_INITIAL_METADATA = 0, OP_SEND_MESSAGE, @@ -60,9 +74,29 @@ enum OP_ID { OP_RECV_TRAILING_METADATA, OP_CANCEL_ERROR, OP_ON_COMPLETE, + OP_FAILED, + OP_CANCELED, + OP_RECV_MESSAGE_AND_ON_COMPLETE, + OP_READ_REQ_MADE, OP_NUM_OPS }; +const char *op_id_string[] = { + "OP_SEND_INITIAL_METADATA", + "OP_SEND_MESSAGE", + "OP_SEND_TRAILING_METADATA", + "OP_RECV_MESSAGE", + "OP_RECV_INITIAL_METADATA", + "OP_RECV_TRAILING_METADATA", + "OP_CANCEL_ERROR", + "OP_ON_COMPLETE", + "OP_FAILED", + "OP_CANCELED", + "OP_RECV_MESSAGE_AND_ON_COMPLETE", + "OP_READ_REQ_MADE", + "OP_NUM_OPS" +}; + /* Cronet callbacks */ static void on_request_headers_sent(cronet_bidirectional_stream *); @@ -75,7 +109,7 @@ static void on_response_trailers_received(cronet_bidirectional_stream *, const cronet_bidirectional_stream_header_array *); static void on_succeeded(cronet_bidirectional_stream *); static void on_failed(cronet_bidirectional_stream *, int); -//static void on_canceled(cronet_bidirectional_stream *); +static void on_canceled(cronet_bidirectional_stream *); static cronet_bidirectional_stream_callback cronet_callbacks = { on_request_headers_sent, on_response_headers_received, @@ -84,7 +118,7 @@ static cronet_bidirectional_stream_callback cronet_callbacks = { on_response_trailers_received, on_succeeded, on_failed, - NULL //on_canceled + on_canceled }; // Cronet transport object @@ -121,30 +155,49 @@ struct write_state { char *write_buffer; }; -#define MAX_PENDING_OPS 10 +// maximum ops in a batch.. There is not much thinking behind this limit, except +// that it seems to be enough for most use cases. +#define MAX_PENDING_OPS 100 + +struct op_state { + bool state_op_done[OP_NUM_OPS]; + bool state_callback_received[OP_NUM_OPS]; + // data structure for storing data coming from server + struct read_state rs; + // data structure for storing data going to the server + struct write_state ws; +}; + +struct op_and_state { + grpc_transport_stream_op op; + struct op_state state; + bool done; + struct stream_obj *s; // Pointer back to the stream object +}; + struct op_storage { - grpc_transport_stream_op pending_ops[MAX_PENDING_OPS]; + struct op_and_state pending_ops[MAX_PENDING_OPS]; int wrptr; int rdptr; int num_pending_ops; }; struct stream_obj { + struct op_and_state *oas; grpc_transport_stream_op *curr_op; grpc_cronet_transport curr_ct; grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; - // TODO (makdharma) : make a sub structure for tracking state - bool state_op_done[OP_NUM_OPS]; - bool state_callback_received[OP_NUM_OPS]; + // This holds the state that is at stream level (response and req metadata) + struct op_state state; - // Read state - struct read_state rs; - // Write state - struct write_state ws; + //struct op_state state; // OP storage struct op_storage storage; + + // Mutex to protect execute_curr_streaming_op + gpr_mu mu; }; typedef struct stream_obj stream_obj; @@ -152,123 +205,155 @@ typedef struct stream_obj stream_obj; cronet_bidirectional_stream_header_array header_array; // -static void execute_curr_stream_op(stream_obj *s); +static enum OP_RESULT execute_stream_op(struct op_and_state *oas); /************************************************************* Op Storage */ -static void add_pending_op(struct op_storage *storage, grpc_transport_stream_op *op) { +static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { + struct op_storage *storage = &s->storage; GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); storage->num_pending_ops++; - gpr_log(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", + CRONET_LOG(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", storage->wrptr, storage->num_pending_ops); - memcpy(&storage->pending_ops[storage->wrptr], op, sizeof(grpc_transport_stream_op)); + memcpy(&storage->pending_ops[storage->wrptr].op, op, sizeof(grpc_transport_stream_op)); + memset(&storage->pending_ops[storage->wrptr].state, 0, sizeof(storage->pending_ops[storage->wrptr].state)); + storage->pending_ops[storage->wrptr].done = false; + storage->pending_ops[storage->wrptr].s = s; storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; } -static grpc_transport_stream_op *pop_pending_op(struct op_storage *storage) { - if (storage->num_pending_ops == 0) return NULL; - grpc_transport_stream_op *result = &storage->pending_ops[storage->rdptr]; - storage->rdptr = (storage->rdptr + 1) % MAX_PENDING_OPS; - storage->num_pending_ops--; - gpr_log(GPR_DEBUG, "popping op @rdptr=%d. %d more left in queue", - storage->rdptr, storage->num_pending_ops); - return result; +static void execute_from_storage(stream_obj *s) { + // Cycle through ops and try to take next action. Break when either + // an action with callback is taken, or no action is possible. + gpr_mu_lock(&s->mu); + for (int i = 0; i < s->storage.wrptr; ) { + CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, s->storage.pending_ops[i].done); + if (s->storage.pending_ops[i].done) { + i++; + continue; + } + enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); + CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", OP_RESULT_STRING[result], i); + if (result == NO_ACTION_POSSIBLE) { + i++; + } else if (result == ACTION_TAKEN_WITH_CALLBACK) { + break; + } + } + gpr_mu_unlock(&s->mu); } + /************************************************************* Cronet Callback Ipmlementation */ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { - gpr_log(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); + CRONET_LOG(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); + stream_obj *s = (stream_obj *)stream->annotation; + cronet_bidirectional_stream_destroy(s->cbs); + s->state.state_callback_received[OP_FAILED] = true; + s->cbs = NULL; + execute_from_storage(s); } -static void on_succeeded(cronet_bidirectional_stream *stream) { - gpr_log(GPR_DEBUG, "on_succeeded(%p)", stream); +static void on_canceled(cronet_bidirectional_stream *stream) { + CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream); + stream_obj *s = (stream_obj *)stream->annotation; + cronet_bidirectional_stream_destroy(s->cbs); + s->state.state_callback_received[OP_CANCELED] = true; + s->cbs = NULL; + execute_from_storage(s); } +static void on_succeeded(cronet_bidirectional_stream *stream) { + CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); + stream_obj *s = (stream_obj *)stream->annotation; + cronet_bidirectional_stream_destroy(s->cbs); + s->cbs = NULL; +} static void on_request_headers_sent(cronet_bidirectional_stream *stream) { - gpr_log(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); + CRONET_LOG(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; - s->state_op_done[OP_SEND_INITIAL_METADATA] = true; - s->state_callback_received[OP_SEND_INITIAL_METADATA] = true; - execute_curr_stream_op(s); + s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true; + s->state.state_callback_received[OP_SEND_INITIAL_METADATA] = true; + execute_from_storage(s); } static void on_response_headers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { - gpr_log(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, + CRONET_LOG(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, headers, negotiated_protocol); - stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->rs.initial_metadata, 0, sizeof(s->rs.initial_metadata)); - grpc_chttp2_incoming_metadata_buffer_init(&s->rs.initial_metadata); + memset(&s->state.rs.initial_metadata, 0, sizeof(s->state.rs.initial_metadata)); + grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata); unsigned int i = 0; for (i = 0; i < headers->count; i++) { grpc_chttp2_incoming_metadata_buffer_add( - &s->rs.initial_metadata, + &s->state.rs.initial_metadata, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(headers->headers[i].key), grpc_mdstr_from_string(headers->headers[i].value))); } - s->state_callback_received[OP_RECV_INITIAL_METADATA] = true; - execute_curr_stream_op(s); + s->state.state_callback_received[OP_RECV_INITIAL_METADATA] = true; + execute_from_storage(s); } static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { - gpr_log(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); stream_obj *s = (stream_obj *)stream->annotation; - if (s->ws.write_buffer) { - gpr_free(s->ws.write_buffer); - s->ws.write_buffer = NULL; + CRONET_LOG(GPR_DEBUG, "W: on_write_completed(%p, %s)", stream, data); + if (s->state.ws.write_buffer) { + gpr_free(s->state.ws.write_buffer); + s->state.ws.write_buffer = NULL; } - s->state_callback_received[OP_SEND_MESSAGE] = true; - execute_curr_stream_op(s); + s->state.state_callback_received[OP_SEND_MESSAGE] = true; + execute_from_storage(s); } static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; - gpr_log(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); if (count > 0) { - s->rs.received_bytes += count; - s->rs.remaining_bytes -= count; - if (s->rs.remaining_bytes > 0) { - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); - cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer + s->rs.received_bytes, s->rs.remaining_bytes); + s->state.rs.received_bytes += count; + s->state.rs.remaining_bytes -= count; + if (s->state.rs.remaining_bytes > 0) { + //CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + s->state.state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + cronet_bidirectional_stream_read(s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, s->state.rs.remaining_bytes); } else { - execute_curr_stream_op(s); + execute_from_storage(s); } - s->state_callback_received[OP_RECV_MESSAGE] = true; + s->state.state_callback_received[OP_RECV_MESSAGE] = true; } } static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { - gpr_log(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, + CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, trailers); stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->rs.trailing_metadata, 0, sizeof(s->rs.trailing_metadata)); - s->rs.trailing_metadata_valid = false; - grpc_chttp2_incoming_metadata_buffer_init(&s->rs.trailing_metadata); + memset(&s->state.rs.trailing_metadata, 0, sizeof(s->state.rs.trailing_metadata)); + s->state.rs.trailing_metadata_valid = false; + grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata); unsigned int i = 0; for (i = 0; i < trailers->count; i++) { - gpr_log(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, + CRONET_LOG(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, trailers->headers[i].value); grpc_chttp2_incoming_metadata_buffer_add( - &s->rs.trailing_metadata, grpc_mdelem_from_metadata_strings( + &s->state.rs.trailing_metadata, grpc_mdelem_from_metadata_strings( grpc_mdstr_from_string(trailers->headers[i].key), grpc_mdstr_from_string(trailers->headers[i].value))); - s->rs.trailing_metadata_valid = true; + s->state.rs.trailing_metadata_valid = true; } - s->state_callback_received[OP_RECV_TRAILING_METADATA] = true; - execute_curr_stream_op(s); + s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true; + execute_from_storage(s); } /************************************************************* @@ -295,10 +380,10 @@ static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, memcpy(p, GPR_SLICE_START_PTR(slice), length); } -static void enqueue_callback(grpc_closure *callback) { +static void enqueue_callback(grpc_closure *callback, grpc_error *error) { GPR_ASSERT(callback); grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_exec_ctx_sched(&exec_ctx, callback, GRPC_ERROR_NONE, NULL); + grpc_exec_ctx_sched(&exec_ctx, callback, error, NULL); grpc_exec_ctx_finish(&exec_ctx); } @@ -342,6 +427,7 @@ static void convert_metadata_to_cronet_headers( gpr_asprintf(pp_url, "https://%s%s", host, value); continue; } + gpr_log(GPR_DEBUG, "header %s = %s", key, value); headers[num_headers].key = key; headers[num_headers].value = value; num_headers++; @@ -365,165 +451,252 @@ static int parse_grpc_header(const uint8_t *data) { /* Op Execution */ - -static bool op_can_be_run(stream_obj *s, enum OP_ID op_id) { - if (op_id == OP_SEND_INITIAL_METADATA) { +static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *stream_state, struct op_state *op_state, enum OP_ID op_id) { + // We use op's own state for most state, except for metadata related callbacks, which + // are at the stream level. TODO: WTF does this comment mean? + bool result = true; + // When call is canceled, every op can be run + if (stream_state->state_op_done[OP_CANCEL_ERROR] || stream_state->state_callback_received[OP_FAILED]) { + if (op_id == OP_SEND_INITIAL_METADATA) result = false; + if (op_id == OP_SEND_MESSAGE) result = false; + if (op_id == OP_SEND_TRAILING_METADATA) result = false; + if (op_id == OP_CANCEL_ERROR) result = false; // already executed - if (s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; - } - if (op_id == OP_RECV_INITIAL_METADATA) { + if (op_id == OP_RECV_INITIAL_METADATA && stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; + if (op_id == OP_RECV_MESSAGE && stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; + if (op_id == OP_RECV_TRAILING_METADATA && stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + } else if (op_id == OP_SEND_INITIAL_METADATA) { + // already executed + if (stream_state->state_op_done[OP_SEND_INITIAL_METADATA]) result = false; + } else if (op_id == OP_RECV_INITIAL_METADATA) { // already executed - if (s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; + if (stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; // we haven't sent headers yet. - if (!s->state_callback_received[OP_SEND_INITIAL_METADATA]) return false; + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; // we haven't received headers yet. - if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; - } - if (op_id == OP_SEND_MESSAGE) { + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + } else if (op_id == OP_SEND_MESSAGE) { // already executed - if (s->state_op_done[OP_SEND_MESSAGE]) return false; - // we haven't received headers yet. - if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; - } - if (op_id == OP_RECV_MESSAGE) { + if (stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + // we haven't sent headers yet. + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; + } else if (op_id == OP_RECV_MESSAGE) { // already executed - if (s->state_op_done[OP_RECV_MESSAGE]) return false; + if (op_state->state_op_done[OP_RECV_MESSAGE]) result = false; // we haven't received headers yet. - if (!s->state_callback_received[OP_RECV_INITIAL_METADATA]) return false; - } - if (op_id == OP_RECV_TRAILING_METADATA) { + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + } else if (op_id == OP_RECV_TRAILING_METADATA) { // already executed - if (s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + if (stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + // we have asked for but haven't received message yet. + else if (stream_state->state_op_done[OP_READ_REQ_MADE] && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; // we haven't received trailers yet. - if (!s->state_callback_received[OP_RECV_TRAILING_METADATA]) return false; - } - if (op_id == OP_SEND_TRAILING_METADATA) { + else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) result = false; + } else if (op_id == OP_SEND_TRAILING_METADATA) { // already executed - if (s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; + if (stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; + // we haven't sent initial metadata yet + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; // we haven't sent message yet - if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; - } - - if (op_id == OP_ON_COMPLETE) { + else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + // we haven't got on_write_completed for the send yet + else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + } else if (op_id == OP_CANCEL_ERROR) { // already executed - if (s->state_op_done[OP_ON_COMPLETE]) return false; + if (stream_state->state_op_done[OP_CANCEL_ERROR]) result = false; + } else if (op_id == OP_ON_COMPLETE) { + // already executed (note we're checking op specific state, not stream state) + if (op_state->state_op_done[OP_ON_COMPLETE]) result = false; // Check if every op that was asked for is done. - if (s->curr_op->send_initial_metadata && !s->state_op_done[OP_SEND_INITIAL_METADATA]) return false; - if (s->curr_op->send_message && !s->state_op_done[OP_SEND_MESSAGE]) return false; - if (s->curr_op->send_trailing_metadata && !s->state_op_done[OP_SEND_TRAILING_METADATA]) return false; - if (s->curr_op->recv_initial_metadata && !s->state_op_done[OP_RECV_INITIAL_METADATA]) return false; - if (s->curr_op->recv_message && !s->state_op_done[OP_RECV_MESSAGE]) return false; - if (s->curr_op->recv_trailing_metadata && !s->state_op_done[OP_RECV_TRAILING_METADATA]) return false; + else if (curr_op->send_initial_metadata && !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; + else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + else if (curr_op->send_trailing_metadata && !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; + else if (curr_op->recv_initial_metadata && !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; + else if (curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; + else if (curr_op->recv_trailing_metadata) { + // We aren't done with trailing metadata yet + if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + // We've asked for actual message in an earlier op, and it hasn't been delivered yet. + // (TODO: What happens when multiple messages are asked for? How do you know when last message arrived?) + else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { + // If this op is not the one asking for read, (which means some earlier op has asked), and the + // read hasn't been delivered. + if(!curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE]) result = false; + } + } + // We should see at least one on_write_completed for the trailers that we sent + else if (curr_op->send_trailing_metadata && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; } - return true; + CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], result? "YES":"NO"); + return result; } -static void execute_curr_stream_op(stream_obj *s) { - if (s->curr_op->send_initial_metadata && op_can_be_run(s, OP_SEND_INITIAL_METADATA)) { +static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { + // TODO TODO : This can be called from network thread and main thread. add a mutex. + grpc_transport_stream_op *stream_op = &oas->op; + struct stream_obj *s = oas->s; + struct op_state *stream_state = &s->state; + //CRONET_LOG(GPR_DEBUG, "execute_stream_op"); + enum OP_RESULT result = NO_ACTION_POSSIBLE; + if (stream_op->send_initial_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_INITIAL_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); // This OP is the beginning. Reset various states - memset(&s->rs, 0, sizeof(s->rs)); - memset(&s->ws, 0, sizeof(s->ws)); - memset(s->state_op_done, 0, sizeof(s->state_op_done)); - memset(s->state_callback_received, 0, sizeof(s->state_callback_received)); + memset(&stream_state->rs, 0, sizeof(stream_state->rs)); + memset(&stream_state->ws, 0, sizeof(stream_state->ws)); + memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done)); + memset(stream_state->state_callback_received, 0, sizeof(stream_state->state_callback_received)); // Start new cronet stream GPR_ASSERT(s->cbs == NULL); - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_create"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_create"); s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); char *url; - convert_metadata_to_cronet_headers(s->curr_op->send_initial_metadata->list.head, + convert_metadata_to_cronet_headers(stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, &header_array.headers, &header_array.count); header_array.capacity = header_array.count; - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_start"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_start %s", url); cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &header_array, false); - s->state_op_done[OP_SEND_INITIAL_METADATA] = true; - } else if (s->curr_op->recv_initial_metadata && - op_can_be_run(s, OP_RECV_INITIAL_METADATA)) { - grpc_chttp2_incoming_metadata_buffer_publish(&s->rs.initial_metadata, - s->curr_op->recv_initial_metadata); - enqueue_callback(s->curr_op->recv_initial_metadata_ready); - s->state_op_done[OP_RECV_INITIAL_METADATA] = true; - // We are ready to execute send_message. - execute_curr_stream_op(s); - } else if (s->curr_op->send_message && op_can_be_run(s, OP_SEND_MESSAGE)) { + stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_op->recv_initial_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_INITIAL_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); + if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { + grpc_chttp2_incoming_metadata_buffer_publish(&oas->s->state.rs.initial_metadata, + stream_op->recv_initial_metadata); + enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE); + } else { + enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED); + } + stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; + result = ACTION_TAKEN_NO_CALLBACK; + } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_MESSAGE)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); // TODO (makdharma): Make into a standalone function gpr_slice_buffer write_slice_buffer; gpr_slice slice; gpr_slice_buffer_init(&write_slice_buffer); - grpc_byte_stream_next(NULL, s->curr_op->send_message, &slice, - s->curr_op->send_message->length, NULL); + grpc_byte_stream_next(NULL, stream_op->send_message, &slice, + stream_op->send_message->length, NULL); // Check that compression flag is not ON. We don't support compression yet. // TODO (makdharma): add compression support - GPR_ASSERT(s->curr_op->send_message->flags == 0); + GPR_ASSERT(stream_op->send_message->flags == 0); gpr_slice_buffer_add(&write_slice_buffer, slice); GPR_ASSERT(write_slice_buffer.count == 1); // Empty request not handled yet if (write_slice_buffer.count > 0) { int write_buffer_size; - create_grpc_frame(&write_slice_buffer, &s->ws.write_buffer, &write_buffer_size); - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", s->ws.write_buffer); - cronet_bidirectional_stream_write(s->cbs, s->ws.write_buffer, + create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, &write_buffer_size); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", stream_state->ws.write_buffer); + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, write_buffer_size, false); // TODO: What if this is not the last write? + result = ACTION_TAKEN_WITH_CALLBACK; } - s->state_op_done[OP_SEND_MESSAGE] = true; - } else if (s->curr_op->recv_message && op_can_be_run(s, OP_RECV_MESSAGE)) { - if (s->rs.length_field_received == false) { - if (s->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && s->rs.remaining_bytes == 0) { + stream_state->state_op_done[OP_SEND_MESSAGE] = true; + } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); + if (stream_state->state_op_done[OP_CANCEL_ERROR]) { + enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_CANCELLED); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + } else if (stream_state->rs.length_field_received == false) { + if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { // Start a read operation for data - s->rs.length_field_received = true; - s->rs.length_field = s->rs.remaining_bytes = - parse_grpc_header((const uint8_t *)s->rs.read_buffer); - GPR_ASSERT(s->rs.length_field > 0); // Empty message? - gpr_log(GPR_DEBUG, "length field = %d", s->rs.length_field); - s->rs.read_buffer = gpr_malloc(s->rs.length_field); - GPR_ASSERT(s->rs.read_buffer); - s->rs.remaining_bytes = s->rs.length_field; - s->rs.received_bytes = 0; - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); - cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, - s->rs.remaining_bytes); - } else if (s->rs.remaining_bytes == 0) { + stream_state->rs.length_field_received = true; + stream_state->rs.length_field = stream_state->rs.remaining_bytes = + parse_grpc_header((const uint8_t *)stream_state->rs.read_buffer); + CRONET_LOG(GPR_DEBUG, "length field = %d", stream_state->rs.length_field); + if (stream_state->rs.length_field > 0) { + stream_state->rs.read_buffer = gpr_malloc(stream_state->rs.length_field); + GPR_ASSERT(stream_state->rs.read_buffer); + stream_state->rs.remaining_bytes = stream_state->rs.length_field; + stream_state->rs.received_bytes = 0; + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, + stream_state->rs.remaining_bytes); + result = ACTION_TAKEN_WITH_CALLBACK; + } else { + stream_state->rs.remaining_bytes = 0; + CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); + gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; + enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. + result = ACTION_TAKEN_NO_CALLBACK; + } + } else if (stream_state->rs.remaining_bytes == 0) { // Start a read operation for first 5 bytes (GRPC header) - s->rs.read_buffer = s->rs.grpc_header_bytes; - s->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; - s->rs.received_bytes = 0; - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_read"); - cronet_bidirectional_stream_read(s->cbs, s->rs.read_buffer, - s->rs.remaining_bytes); + stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; + stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; + stream_state->rs.received_bytes = 0; + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, + stream_state->rs.remaining_bytes); } - } else if (s->rs.remaining_bytes == 0) { - gpr_log(GPR_DEBUG, "read operation complete"); - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->rs.length_field); + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_state->rs.remaining_bytes == 0) { + CRONET_LOG(GPR_DEBUG, "read operation complete"); + gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)stream_state->rs.length_field); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, s->rs.read_buffer, (size_t)s->rs.length_field); - gpr_slice_buffer_init(&s->rs.read_slice_buffer); - gpr_slice_buffer_add(&s->rs.read_slice_buffer, read_data_slice); - grpc_slice_buffer_stream_init(&s->rs.sbs, &s->rs.read_slice_buffer, 0); - *((grpc_byte_buffer **)s->curr_op->recv_message) = (grpc_byte_buffer *)&s->rs.sbs; - enqueue_callback(s->curr_op->recv_message_ready); - s->state_op_done[OP_RECV_MESSAGE] = true; - execute_curr_stream_op(s); + memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); + gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); + gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; + enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. + // Clear read state of the stream, so next read op (if it were to come) will work + stream_state->rs.received_bytes = stream_state->rs.remaining_bytes = stream_state->rs.length_field_received = 0; + result = ACTION_TAKEN_NO_CALLBACK; } - } else if (s->curr_op->recv_trailing_metadata && - op_can_be_run(s, OP_RECV_TRAILING_METADATA)) { - if (s->rs.trailing_metadata_valid) { + } else if (stream_op->recv_trailing_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_TRAILING_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_TRAILING_METADATA", oas); + if (oas->s->state.rs.trailing_metadata_valid) { grpc_chttp2_incoming_metadata_buffer_publish( - &s->rs.trailing_metadata, s->curr_op->recv_trailing_metadata); - s->rs.trailing_metadata_valid = false; + &oas->s->state.rs.trailing_metadata, stream_op->recv_trailing_metadata); + stream_state->rs.trailing_metadata_valid = false; } - s->state_op_done[OP_RECV_TRAILING_METADATA] = true; - execute_curr_stream_op(s); - } else if (s->curr_op->send_trailing_metadata && - op_can_be_run(s, OP_SEND_TRAILING_METADATA)) { - - gpr_log(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; + result = ACTION_TAKEN_NO_CALLBACK; + } else if (stream_op->send_trailing_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_TRAILING_METADATA)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, "", 0, true); - s->state_op_done[OP_SEND_TRAILING_METADATA] = true; - } else if (op_can_be_run(s, OP_ON_COMPLETE)) { + stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_op->cancel_error && op_can_be_run(stream_op, stream_state, &oas->state, OP_CANCEL_ERROR)) { + CRONET_LOG(GPR_DEBUG, "running: %p OP_CANCEL_ERROR", oas); + CRONET_LOG(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); + // Cancel might have come before creation of stream + if (s->cbs) { + cronet_bidirectional_stream_cancel(s->cbs); + } + stream_state->state_op_done[OP_CANCEL_ERROR] = true; + result = ACTION_TAKEN_WITH_CALLBACK; + } else if (stream_op->on_complete && op_can_be_run(stream_op, stream_state, &oas->state, OP_ON_COMPLETE)) { // All ops are complete. Call the on_complete callback - enqueue_callback(s->curr_op->on_complete); - s->state_op_done[OP_ON_COMPLETE] = true; - cronet_bidirectional_stream_destroy(s->cbs); - s->cbs = NULL; + CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); + //CRONET_LOG(GPR_DEBUG, "calling on_complete"); + enqueue_callback(stream_op->on_complete, GRPC_ERROR_NONE); + // Instead of setting stream state, use the op state as on_complete is on per op basis + oas->state.state_op_done[OP_ON_COMPLETE] = true; + oas->done = true; // Mark this op as completed + // reset any send or receive message state. + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + stream_state->state_op_done[OP_SEND_MESSAGE] = false; + result = ACTION_TAKEN_NO_CALLBACK; + // If this is the on_complete callback being called for a received message - make a note + if (stream_op->recv_message) stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; + } else { + //CRONET_LOG(GPR_DEBUG, "No op ready to run"); + result = NO_ACTION_POSSIBLE; } + return result; } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -533,7 +706,14 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, const void *server_data) { stream_obj *s = (stream_obj *)gs; memset(&s->storage, 0, sizeof(s->storage)); + memset(&s->state, 0, sizeof(s->state)); s->curr_op = NULL; + s->cbs = NULL; + memset(&s->state.rs, 0, sizeof(s->state.rs)); + memset(&s->state.ws, 0, sizeof(s->state.ws)); + memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); + memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); + gpr_mu_init(&s->mu); return 0; } @@ -546,15 +726,12 @@ static void set_pollset_set_do_nothing(grpc_exec_ctx *exec_ctx, static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_transport_stream_op *op) { - gpr_log(GPR_DEBUG, "perform_stream_op"); + CRONET_LOG(GPR_DEBUG, "perform_stream_op"); stream_obj *s = (stream_obj *)gs; - memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); - add_pending_op(&s->storage, op); - if (s->curr_op == NULL) { - s->curr_op = pop_pending_op(&s->storage); - } s->curr_gs = gs; - execute_curr_stream_op(s); + memcpy(&s->curr_ct, gt, sizeof(grpc_cronet_transport)); + add_to_storage(s, op); + execute_from_storage(s); } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, From 198f8b01946ac107700958ac6c2ea11ed523f2a5 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 10 Aug 2016 13:44:13 -0700 Subject: [PATCH 25/43] more fixes --- .../cronet/transport/cronet_transport.c | 42 +++++++++++-------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index e8746b4e6e6..dc6a780edaf 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -75,6 +75,7 @@ enum OP_ID { OP_CANCEL_ERROR, OP_ON_COMPLETE, OP_FAILED, + OP_SUCCEEDED, OP_CANCELED, OP_RECV_MESSAGE_AND_ON_COMPLETE, OP_READ_REQ_MADE, @@ -91,6 +92,7 @@ const char *op_id_string[] = { "OP_CANCEL_ERROR", "OP_ON_COMPLETE", "OP_FAILED", + "OP_SUCCEEDED", "OP_CANCELED", "OP_RECV_MESSAGE_AND_ON_COMPLETE", "OP_READ_REQ_MADE", @@ -189,6 +191,8 @@ struct stream_obj { grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; + // Used for executing callbacks for ops + grpc_exec_ctx exec_ctx; // This holds the state that is at stream level (response and req metadata) struct op_state state; @@ -227,7 +231,10 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { static void execute_from_storage(stream_obj *s) { // Cycle through ops and try to take next action. Break when either // an action with callback is taken, or no action is possible. - gpr_mu_lock(&s->mu); + // This can be executed from the Cronet network thread via cronet callback + // or on the application supplied thread via the perform_stream_op function. + if (1) {//gpr_mu_lock(&s->mu) == 0) { + gpr_mu_lock(&s->mu); for (int i = 0; i < s->storage.wrptr; ) { CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, s->storage.pending_ops[i].done); if (s->storage.pending_ops[i].done) { @@ -242,7 +249,9 @@ static void execute_from_storage(stream_obj *s) { break; } } - gpr_mu_unlock(&s->mu); + gpr_mu_unlock(&s->mu); + } + grpc_exec_ctx_finish(&s->exec_ctx); } @@ -271,7 +280,9 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; cronet_bidirectional_stream_destroy(s->cbs); + s->state.state_callback_received[OP_FAILED] = true; s->cbs = NULL; + execute_from_storage(s); } static void on_request_headers_sent(cronet_bidirectional_stream *stream) { @@ -380,13 +391,6 @@ static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, memcpy(p, GPR_SLICE_START_PTR(slice), length); } -static void enqueue_callback(grpc_closure *callback, grpc_error *error) { - GPR_ASSERT(callback); - grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; - grpc_exec_ctx_sched(&exec_ctx, callback, error, NULL); - grpc_exec_ctx_finish(&exec_ctx); -} - static void convert_metadata_to_cronet_headers( grpc_linked_mdelem *head, const char *host, @@ -498,9 +502,10 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st // we haven't sent initial metadata yet else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; // we haven't sent message yet + // TODO: Streaming Write case is a problem. What if there is an outstanding write (2nd, 3rd,..) present. else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; // we haven't got on_write_completed for the send yet - else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + else if (stream_state->state_op_done[OP_SEND_MESSAGE] && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; } else if (op_id == OP_CANCEL_ERROR) { // already executed if (stream_state->state_op_done[OP_CANCEL_ERROR]) result = false; @@ -510,10 +515,12 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st // Check if every op that was asked for is done. else if (curr_op->send_initial_metadata && !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; else if (curr_op->send_trailing_metadata && !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; else if (curr_op->recv_initial_metadata && !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; else if (curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; else if (curr_op->recv_trailing_metadata) { + //if (!stream_state->state_op_done[OP_SUCCEEDED]) result = false; gpr_log(GPR_DEBUG, "HACK!!"); // We aren't done with trailing metadata yet if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; // We've asked for actual message in an earlier op, and it hasn't been delivered yet. @@ -521,7 +528,7 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { // If this op is not the one asking for read, (which means some earlier op has asked), and the // read hasn't been delivered. - if(!curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE]) result = false; + if(!curr_op->recv_message && !stream_state->state_op_done[OP_SUCCEEDED]) result = false; } } // We should see at least one on_write_completed for the trailers that we sent @@ -563,9 +570,9 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { grpc_chttp2_incoming_metadata_buffer_publish(&oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); - enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE, NULL); } else { - enqueue_callback(stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED, NULL); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; @@ -595,7 +602,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_CANCELLED); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.length_field_received == false) { if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { @@ -620,7 +627,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. result = ACTION_TAKEN_NO_CALLBACK; @@ -645,7 +652,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - enqueue_callback(stream_op->recv_message_ready, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. // Clear read state of the stream, so next read op (if it were to come) will work @@ -682,7 +689,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { // All ops are complete. Call the on_complete callback CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); //CRONET_LOG(GPR_DEBUG, "calling on_complete"); - enqueue_callback(stream_op->on_complete, GRPC_ERROR_NONE); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, NULL); // Instead of setting stream state, use the op state as on_complete is on per op basis oas->state.state_op_done[OP_ON_COMPLETE] = true; oas->done = true; // Mark this op as completed @@ -714,6 +721,7 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); gpr_mu_init(&s->mu); + s->exec_ctx = *exec_ctx; return 0; } From 35da822b442a60d5e9d0711cc046f9c578d63ad9 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 10 Aug 2016 14:53:40 -0700 Subject: [PATCH 26/43] WIP --- src/core/ext/transport/cronet/transport/cronet_transport.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index dc6a780edaf..7e65def4de1 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -280,7 +280,7 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; cronet_bidirectional_stream_destroy(s->cbs); - s->state.state_callback_received[OP_FAILED] = true; + s->state.state_callback_received[OP_SUCCEEDED] = true; s->cbs = NULL; execute_from_storage(s); } @@ -480,8 +480,8 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *st // we haven't received headers yet. else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; } else if (op_id == OP_SEND_MESSAGE) { - // already executed - if (stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; + // already executed (note we're checking op specific state, not stream state) + if (op_state->state_op_done[OP_SEND_MESSAGE]) result = false; // we haven't sent headers yet. else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; } else if (op_id == OP_RECV_MESSAGE) { @@ -599,6 +599,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_MESSAGE] = true; + oas->state.state_op_done[OP_SEND_MESSAGE] = true; } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { From f66a37b63aed77732c8d7d253af89296a23bb72e Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 10 Aug 2016 17:47:25 -0700 Subject: [PATCH 27/43] WIP --- .../cronet/transport/cronet_transport.c | 597 +++++++++++------- 1 file changed, 356 insertions(+), 241 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 7e65def4de1..1d756039809 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -50,8 +50,17 @@ #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" #define GRPC_HEADER_SIZE_IN_BYTES 5 +// maximum ops in a batch. There is not much thinking behind this limit, except +// that it seems to be enough for most use cases. +#define MAX_PENDING_OPS 100 + +#define CRONET_LOG(...) \ + { \ + if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ + } -#define CRONET_LOG(...) {if (1) gpr_log(__VA_ARGS__);} +// TODO (makdharma): Hook up into the wider tracing mechanism +int grpc_cronet_trace = 1; enum OP_RESULT { ACTION_TAKEN_WITH_CALLBACK, @@ -59,11 +68,10 @@ enum OP_RESULT { NO_ACTION_POSSIBLE }; -const char *OP_RESULT_STRING[] = { - "ACTION_TAKEN_WITH_CALLBACK", - "ACTION_TAKEN_NO_CALLBACK", - "NO_ACTION_POSSIBLE" -}; +// Used for printing debug +const char *op_result_string[] = {"ACTION_TAKEN_WITH_CALLBACK", + "ACTION_TAKEN_NO_CALLBACK", + "NO_ACTION_POSSIBLE"}; enum OP_ID { OP_SEND_INITIAL_METADATA = 0, @@ -82,32 +90,31 @@ enum OP_ID { OP_NUM_OPS }; -const char *op_id_string[] = { - "OP_SEND_INITIAL_METADATA", - "OP_SEND_MESSAGE", - "OP_SEND_TRAILING_METADATA", - "OP_RECV_MESSAGE", - "OP_RECV_INITIAL_METADATA", - "OP_RECV_TRAILING_METADATA", - "OP_CANCEL_ERROR", - "OP_ON_COMPLETE", - "OP_FAILED", - "OP_SUCCEEDED", - "OP_CANCELED", - "OP_RECV_MESSAGE_AND_ON_COMPLETE", - "OP_READ_REQ_MADE", - "OP_NUM_OPS" -}; +const char *op_id_string[] = {"OP_SEND_INITIAL_METADATA", + "OP_SEND_MESSAGE", + "OP_SEND_TRAILING_METADATA", + "OP_RECV_MESSAGE", + "OP_RECV_INITIAL_METADATA", + "OP_RECV_TRAILING_METADATA", + "OP_CANCEL_ERROR", + "OP_ON_COMPLETE", + "OP_FAILED", + "OP_SUCCEEDED", + "OP_CANCELED", + "OP_RECV_MESSAGE_AND_ON_COMPLETE", + "OP_READ_REQ_MADE", + "OP_NUM_OPS"}; /* Cronet callbacks */ static void on_request_headers_sent(cronet_bidirectional_stream *); -static void on_response_headers_received(cronet_bidirectional_stream *, - const cronet_bidirectional_stream_header_array *, - const char *); +static void on_response_headers_received( + cronet_bidirectional_stream *, + const cronet_bidirectional_stream_header_array *, const char *); static void on_write_completed(cronet_bidirectional_stream *, const char *); static void on_read_completed(cronet_bidirectional_stream *, char *, int); -static void on_response_trailers_received(cronet_bidirectional_stream *, +static void on_response_trailers_received( + cronet_bidirectional_stream *, const cronet_bidirectional_stream_header_array *); static void on_succeeded(cronet_bidirectional_stream *); static void on_failed(cronet_bidirectional_stream *, int); @@ -120,8 +127,7 @@ static cronet_bidirectional_stream_callback cronet_callbacks = { on_response_trailers_received, on_succeeded, on_failed, - on_canceled -}; + on_canceled}; // Cronet transport object struct grpc_cronet_transport { @@ -132,7 +138,7 @@ struct grpc_cronet_transport { typedef struct grpc_cronet_transport grpc_cronet_transport; struct read_state { - // vars to store data coming from cronet + /* vars to store data coming from server */ char *read_buffer; bool length_field_received; int received_bytes; @@ -141,15 +147,15 @@ struct read_state { char grpc_header_bytes[GRPC_HEADER_SIZE_IN_BYTES]; char *payload_field; - // vars for holding data destined for the application + /* vars for holding data destined for the application */ struct grpc_slice_buffer_stream sbs; gpr_slice_buffer read_slice_buffer; - // vars for trailing metadata + /* vars for trailing metadata */ grpc_chttp2_incoming_metadata_buffer trailing_metadata; bool trailing_metadata_valid; - // vars for initial metadata + /* vars for initial metadata */ grpc_chttp2_incoming_metadata_buffer initial_metadata; }; @@ -157,16 +163,13 @@ struct write_state { char *write_buffer; }; -// maximum ops in a batch.. There is not much thinking behind this limit, except -// that it seems to be enough for most use cases. -#define MAX_PENDING_OPS 100 - +/* track state of one stream op */ struct op_state { bool state_op_done[OP_NUM_OPS]; bool state_callback_received[OP_NUM_OPS]; - // data structure for storing data coming from server + /* data structure for storing data coming from server */ struct read_state rs; - // data structure for storing data going to the server + /* data structure for storing data going to the server */ struct write_state ws; }; @@ -174,7 +177,7 @@ struct op_and_state { grpc_transport_stream_op op; struct op_state state; bool done; - struct stream_obj *s; // Pointer back to the stream object + struct stream_obj *s; /* Pointer back to the stream object */ }; struct op_storage { @@ -190,73 +193,74 @@ struct stream_obj { grpc_cronet_transport curr_ct; grpc_stream *curr_gs; cronet_bidirectional_stream *cbs; + cronet_bidirectional_stream_header_array header_array; - // Used for executing callbacks for ops + /* Used for executing callbacks for ops */ grpc_exec_ctx exec_ctx; - // This holds the state that is at stream level (response and req metadata) + /* Stream level state. Some state will be tracked both at stream and stream_op + * level */ struct op_state state; - //struct op_state state; - // OP storage + /* OP storage */ struct op_storage storage; - // Mutex to protect execute_curr_streaming_op + /* Mutex to protect storage */ gpr_mu mu; }; typedef struct stream_obj stream_obj; -/* Globals */ -cronet_bidirectional_stream_header_array header_array; - -// static enum OP_RESULT execute_stream_op(struct op_and_state *oas); -/************************************************************* - Op Storage +/* + Add a new stream op to op storage. */ - static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { + gpr_mu_lock(&s->mu); struct op_storage *storage = &s->storage; GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); storage->num_pending_ops++; CRONET_LOG(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", - storage->wrptr, storage->num_pending_ops); - memcpy(&storage->pending_ops[storage->wrptr].op, op, sizeof(grpc_transport_stream_op)); - memset(&storage->pending_ops[storage->wrptr].state, 0, sizeof(storage->pending_ops[storage->wrptr].state)); + storage->wrptr, storage->num_pending_ops); + memcpy(&storage->pending_ops[storage->wrptr].op, op, + sizeof(grpc_transport_stream_op)); + memset(&storage->pending_ops[storage->wrptr].state, 0, + sizeof(storage->pending_ops[storage->wrptr].state)); storage->pending_ops[storage->wrptr].done = false; storage->pending_ops[storage->wrptr].s = s; - storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; + storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; + gpr_mu_unlock(&s->mu); } +/* + Cycle through ops and try to take next action. Break when either + an action with callback is taken, or no action is possible. + This can be executed from the Cronet network thread via cronet callback + or on the application supplied thread via the perform_stream_op function. +*/ static void execute_from_storage(stream_obj *s) { - // Cycle through ops and try to take next action. Break when either - // an action with callback is taken, or no action is possible. - // This can be executed from the Cronet network thread via cronet callback - // or on the application supplied thread via the perform_stream_op function. - if (1) {//gpr_mu_lock(&s->mu) == 0) { - gpr_mu_lock(&s->mu); - for (int i = 0; i < s->storage.wrptr; ) { - CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, s->storage.pending_ops[i].done); - if (s->storage.pending_ops[i].done) { - i++; - continue; - } - enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); - CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", OP_RESULT_STRING[result], i); - if (result == NO_ACTION_POSSIBLE) { - i++; - } else if (result == ACTION_TAKEN_WITH_CALLBACK) { - break; - } + gpr_mu_lock(&s->mu); + for (int i = 0; i < s->storage.wrptr;) { + CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, + s->storage.pending_ops[i].done); + if (s->storage.pending_ops[i].done) { + i++; + continue; + } + enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); + CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", + op_result_string[result], i); + if (result == NO_ACTION_POSSIBLE) { + i++; + } else if (result == ACTION_TAKEN_WITH_CALLBACK) { + break; } - gpr_mu_unlock(&s->mu); } + gpr_mu_unlock(&s->mu); grpc_exec_ctx_finish(&s->exec_ctx); } - -/************************************************************* -Cronet Callback Ipmlementation +/* + Cronet callback */ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { CRONET_LOG(GPR_DEBUG, "on_failed(%p, %d)", stream, net_error); @@ -267,6 +271,9 @@ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { execute_from_storage(s); } +/* + Cronet callback +*/ static void on_canceled(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_canceled(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; @@ -276,6 +283,9 @@ static void on_canceled(cronet_bidirectional_stream *stream) { execute_from_storage(s); } +/* + Cronet callback +*/ static void on_succeeded(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "on_succeeded(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; @@ -285,22 +295,33 @@ static void on_succeeded(cronet_bidirectional_stream *stream) { execute_from_storage(s); } +/* + Cronet callback +*/ static void on_request_headers_sent(cronet_bidirectional_stream *stream) { CRONET_LOG(GPR_DEBUG, "W: on_request_headers_sent(%p)", stream); stream_obj *s = (stream_obj *)stream->annotation; s->state.state_op_done[OP_SEND_INITIAL_METADATA] = true; s->state.state_callback_received[OP_SEND_INITIAL_METADATA] = true; + /* Free the memory allocated for headers */ + if (s->header_array.headers) { + gpr_free(s->header_array.headers); + } execute_from_storage(s); } +/* + Cronet callback +*/ static void on_response_headers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *headers, const char *negotiated_protocol) { CRONET_LOG(GPR_DEBUG, "R: on_response_headers_received(%p, %p, %s)", stream, - headers, negotiated_protocol); + headers, negotiated_protocol); stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->state.rs.initial_metadata, 0, sizeof(s->state.rs.initial_metadata)); + memset(&s->state.rs.initial_metadata, 0, + sizeof(s->state.rs.initial_metadata)); grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata); unsigned int i = 0; for (i = 0; i < headers->count; i++) { @@ -314,6 +335,9 @@ static void on_response_headers_received( execute_from_storage(s); } +/* + Cronet callback +*/ static void on_write_completed(cronet_bidirectional_stream *stream, const char *data) { stream_obj *s = (stream_obj *)stream->annotation; @@ -326,17 +350,23 @@ static void on_write_completed(cronet_bidirectional_stream *stream, execute_from_storage(s); } +/* + Cronet callback +*/ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, int count) { stream_obj *s = (stream_obj *)stream->annotation; - CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, + count); if (count > 0) { s->state.rs.received_bytes += count; s->state.rs.remaining_bytes -= count; if (s->state.rs.remaining_bytes > 0) { - //CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); - s->state.state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made - cronet_bidirectional_stream_read(s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, s->state.rs.remaining_bytes); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + s->state.state_op_done[OP_READ_REQ_MADE] = true; + cronet_bidirectional_stream_read( + s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, + s->state.rs.remaining_bytes); } else { execute_from_storage(s); } @@ -344,76 +374,82 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, } } +/* + Cronet callback +*/ static void on_response_trailers_received( cronet_bidirectional_stream *stream, const cronet_bidirectional_stream_header_array *trailers) { CRONET_LOG(GPR_DEBUG, "R: on_response_trailers_received(%p,%p)", stream, - trailers); + trailers); stream_obj *s = (stream_obj *)stream->annotation; - memset(&s->state.rs.trailing_metadata, 0, sizeof(s->state.rs.trailing_metadata)); + memset(&s->state.rs.trailing_metadata, 0, + sizeof(s->state.rs.trailing_metadata)); s->state.rs.trailing_metadata_valid = false; grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata); unsigned int i = 0; for (i = 0; i < trailers->count; i++) { CRONET_LOG(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, - trailers->headers[i].value); + trailers->headers[i].value); grpc_chttp2_incoming_metadata_buffer_add( - &s->state.rs.trailing_metadata, grpc_mdelem_from_metadata_strings( - grpc_mdstr_from_string(trailers->headers[i].key), - grpc_mdstr_from_string(trailers->headers[i].value))); + &s->state.rs.trailing_metadata, + grpc_mdelem_from_metadata_strings( + grpc_mdstr_from_string(trailers->headers[i].key), + grpc_mdstr_from_string(trailers->headers[i].value))); s->state.rs.trailing_metadata_valid = true; } s->state.state_callback_received[OP_RECV_TRAILING_METADATA] = true; execute_from_storage(s); } -/************************************************************* -Utility functions. Can be in their own file +/* + Utility function that takes the data from s->write_slice_buffer and assembles + into a contiguous byte stream with 5 byte gRPC header prepended. */ -// This function takes the data from s->write_slice_buffer and assembles into -// a contiguous byte stream with 5 byte gRPC header prepended. static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, - char **pp_write_buffer, int *p_write_buffer_size) { + char **pp_write_buffer, + int *p_write_buffer_size) { gpr_slice slice = gpr_slice_buffer_take_first(write_slice_buffer); size_t length = GPR_SLICE_LENGTH(slice); - // TODO (makdharma): FREE THIS!! HACK! *p_write_buffer_size = (int)length + GRPC_HEADER_SIZE_IN_BYTES; + /* This is freed in the on_write_completed callback */ char *write_buffer = gpr_malloc(length + GRPC_HEADER_SIZE_IN_BYTES); *pp_write_buffer = write_buffer; uint8_t *p = (uint8_t *)write_buffer; - // Append 5 byte header + /* Append 5 byte header */ *p++ = 0; *p++ = (uint8_t)(length >> 24); *p++ = (uint8_t)(length >> 16); *p++ = (uint8_t)(length >> 8); *p++ = (uint8_t)(length); - // append actual data + /* append actual data */ memcpy(p, GPR_SLICE_START_PTR(slice), length); } +/* + Convert metadata in a format that Cronet can consume +*/ static void convert_metadata_to_cronet_headers( - grpc_linked_mdelem *head, - const char *host, - char **pp_url, - cronet_bidirectional_stream_header **pp_headers, - size_t *p_num_headers) { + grpc_linked_mdelem *head, const char *host, char **pp_url, + cronet_bidirectional_stream_header **pp_headers, size_t *p_num_headers) { grpc_linked_mdelem *curr = head; - // Walk the linked list and get number of header fields + /* Walk the linked list and get number of header fields */ uint32_t num_headers_available = 0; while (curr != NULL) { curr = curr->next; num_headers_available++; } - // Allocate enough memory. TODO (makdharma): FREE MEMORY! HACK HACK - cronet_bidirectional_stream_header *headers = - (cronet_bidirectional_stream_header *)gpr_malloc( - sizeof(cronet_bidirectional_stream_header) * num_headers_available); + /* Allocate enough memory. It is freed in the on_request_headers_sent callback + */ + cronet_bidirectional_stream_header *headers = + (cronet_bidirectional_stream_header *)gpr_malloc( + sizeof(cronet_bidirectional_stream_header) * num_headers_available); *pp_headers = headers; - // Walk the linked list again, this time copying the header fields. - // s->num_headers - // can be less than num_headers_available, as some headers are not used for - // cronet + /* Walk the linked list again, this time copying the header fields. + s->num_headers can be less than num_headers_available, as some headers + are not used for cronet + */ curr = head; int num_headers = 0; while (num_headers < num_headers_available) { @@ -423,15 +459,15 @@ static void convert_metadata_to_cronet_headers( const char *value = grpc_mdstr_as_c_string(mdelem->value); if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || strcmp(key, ":authority") == 0) { - // Cronet populates these fields on its own. + /* Cronet populates these fields on its own */ continue; } if (strcmp(key, ":path") == 0) { - // Create URL by appending :path value to the hostname + /* Create URL by appending :path value to the hostname */ gpr_asprintf(pp_url, "https://%s%s", host, value); continue; } - gpr_log(GPR_DEBUG, "header %s = %s", key, value); + CRONET_LOG(GPR_DEBUG, "header %s = %s", key, value); headers[num_headers].key = key; headers[num_headers].value = value; num_headers++; @@ -453,261 +489,342 @@ static int parse_grpc_header(const uint8_t *data) { } /* -Op Execution + Op Execution: Decide if one of the actions contained in the stream op can be + executed. This is the heart of the state machine. */ -static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *stream_state, struct op_state *op_state, enum OP_ID op_id) { - // We use op's own state for most state, except for metadata related callbacks, which - // are at the stream level. TODO: WTF does this comment mean? +static bool op_can_be_run(grpc_transport_stream_op *curr_op, + struct op_state *stream_state, + struct op_state *op_state, enum OP_ID op_id) { bool result = true; - // When call is canceled, every op can be run - if (stream_state->state_op_done[OP_CANCEL_ERROR] || stream_state->state_callback_received[OP_FAILED]) { + /* When call is canceled, every op can be run, except under following + conditions + */ + if (stream_state->state_op_done[OP_CANCEL_ERROR] || + stream_state->state_callback_received[OP_FAILED]) { if (op_id == OP_SEND_INITIAL_METADATA) result = false; if (op_id == OP_SEND_MESSAGE) result = false; if (op_id == OP_SEND_TRAILING_METADATA) result = false; if (op_id == OP_CANCEL_ERROR) result = false; - // already executed - if (op_id == OP_RECV_INITIAL_METADATA && stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; - if (op_id == OP_RECV_MESSAGE && stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; - if (op_id == OP_RECV_TRAILING_METADATA && stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; + /* already executed */ + if (op_id == OP_RECV_INITIAL_METADATA && + stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) + result = false; + if (op_id == OP_RECV_MESSAGE && + stream_state->state_op_done[OP_RECV_MESSAGE]) + result = false; + if (op_id == OP_RECV_TRAILING_METADATA && + stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) + result = false; } else if (op_id == OP_SEND_INITIAL_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_SEND_INITIAL_METADATA]) result = false; } else if (op_id == OP_RECV_INITIAL_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; - // we haven't sent headers yet. - else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; - // we haven't received headers yet. - else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + /* we haven't sent headers yet. */ + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; + /* we haven't received headers yet. */ + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) + result = false; } else if (op_id == OP_SEND_MESSAGE) { - // already executed (note we're checking op specific state, not stream state) + /* already executed (note we're checking op specific state, not stream + state) */ if (op_state->state_op_done[OP_SEND_MESSAGE]) result = false; - // we haven't sent headers yet. - else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; + /* we haven't sent headers yet. */ + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; } else if (op_id == OP_RECV_MESSAGE) { - // already executed + /* already executed */ if (op_state->state_op_done[OP_RECV_MESSAGE]) result = false; - // we haven't received headers yet. - else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) result = false; + /* we haven't received headers yet. */ + else if (!stream_state->state_callback_received[OP_RECV_INITIAL_METADATA]) + result = false; } else if (op_id == OP_RECV_TRAILING_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; - // we have asked for but haven't received message yet. - else if (stream_state->state_op_done[OP_READ_REQ_MADE] && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; - // we haven't received trailers yet. - else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) result = false; + /* we have asked for but haven't received message yet. */ + else if (stream_state->state_op_done[OP_READ_REQ_MADE] && + !stream_state->state_op_done[OP_RECV_MESSAGE]) + result = false; + /* we haven't received trailers yet. */ + else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) + result = false; } else if (op_id == OP_SEND_TRAILING_METADATA) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; - // we haven't sent initial metadata yet - else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; - // we haven't sent message yet - // TODO: Streaming Write case is a problem. What if there is an outstanding write (2nd, 3rd,..) present. - else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; - // we haven't got on_write_completed for the send yet - else if (stream_state->state_op_done[OP_SEND_MESSAGE] && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + /* we haven't sent initial metadata yet */ + else if (!stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; + /* we haven't sent message yet */ + else if (curr_op->send_message && + !stream_state->state_op_done[OP_SEND_MESSAGE]) + result = false; + /* we haven't got on_write_completed for the send yet */ + else if (stream_state->state_op_done[OP_SEND_MESSAGE] && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) + result = false; } else if (op_id == OP_CANCEL_ERROR) { - // already executed + /* already executed */ if (stream_state->state_op_done[OP_CANCEL_ERROR]) result = false; } else if (op_id == OP_ON_COMPLETE) { - // already executed (note we're checking op specific state, not stream state) + /* already executed (note we're checking op specific state, not stream + state) */ if (op_state->state_op_done[OP_ON_COMPLETE]) result = false; - // Check if every op that was asked for is done. - else if (curr_op->send_initial_metadata && !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) result = false; - else if (curr_op->send_message && !stream_state->state_op_done[OP_SEND_MESSAGE]) result = false; - else if (curr_op->send_message && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; - else if (curr_op->send_trailing_metadata && !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; - else if (curr_op->recv_initial_metadata && !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) result = false; - else if (curr_op->recv_message && !stream_state->state_op_done[OP_RECV_MESSAGE]) result = false; + /* Check if every op that was asked for is done. */ + else if (curr_op->send_initial_metadata && + !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + result = false; + else if (curr_op->send_message && + !stream_state->state_op_done[OP_SEND_MESSAGE]) + result = false; + else if (curr_op->send_message && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) + result = false; + else if (curr_op->send_trailing_metadata && + !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) + result = false; + else if (curr_op->recv_initial_metadata && + !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) + result = false; + else if (curr_op->recv_message && + !stream_state->state_op_done[OP_RECV_MESSAGE]) + result = false; else if (curr_op->recv_trailing_metadata) { - //if (!stream_state->state_op_done[OP_SUCCEEDED]) result = false; gpr_log(GPR_DEBUG, "HACK!!"); - // We aren't done with trailing metadata yet - if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) result = false; - // We've asked for actual message in an earlier op, and it hasn't been delivered yet. - // (TODO: What happens when multiple messages are asked for? How do you know when last message arrived?) + /* We aren't done with trailing metadata yet */ + if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) + result = false; + /* We've asked for actual message in an earlier op, and it hasn't been + delivered yet. */ else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { - // If this op is not the one asking for read, (which means some earlier op has asked), and the - // read hasn't been delivered. - if(!curr_op->recv_message && !stream_state->state_op_done[OP_SUCCEEDED]) result = false; + /* If this op is not the one asking for read, (which means some earlier + op has asked), and the read hasn't been delivered. */ + if (!curr_op->recv_message && + !stream_state->state_op_done[OP_SUCCEEDED]) + result = false; } } - // We should see at least one on_write_completed for the trailers that we sent - else if (curr_op->send_trailing_metadata && !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; + /* We should see at least one on_write_completed for the trailers that we + sent */ + else if (curr_op->send_trailing_metadata && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) + result = false; } - CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], result? "YES":"NO"); + CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], + result ? "YES" : "NO"); return result; } static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { - // TODO TODO : This can be called from network thread and main thread. add a mutex. grpc_transport_stream_op *stream_op = &oas->op; struct stream_obj *s = oas->s; struct op_state *stream_state = &s->state; - //CRONET_LOG(GPR_DEBUG, "execute_stream_op"); enum OP_RESULT result = NO_ACTION_POSSIBLE; - if (stream_op->send_initial_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_INITIAL_METADATA)) { + if (stream_op->send_initial_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); - // This OP is the beginning. Reset various states + /* This OP is the beginning. Reset various states */ memset(&stream_state->rs, 0, sizeof(stream_state->rs)); memset(&stream_state->ws, 0, sizeof(stream_state->ws)); memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done)); - memset(stream_state->state_callback_received, 0, sizeof(stream_state->state_callback_received)); - // Start new cronet stream + memset(stream_state->state_callback_received, 0, + sizeof(stream_state->state_callback_received)); + /* Start new cronet stream. It is destroyed in on_succeeded, on_canceled, + * on_failed */ GPR_ASSERT(s->cbs == NULL); CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_create"); - s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); + s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, + &cronet_callbacks); char *url; - convert_metadata_to_cronet_headers(stream_op->send_initial_metadata->list.head, - s->curr_ct.host, &url, &header_array.headers, &header_array.count); - header_array.capacity = header_array.count; + convert_metadata_to_cronet_headers( + stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, + &s->header_array.headers, &s->header_array.count); + s->header_array.capacity = s->header_array.count; CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_start %s", url); - cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &header_array, false); + cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &s->header_array, + false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_op->recv_initial_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_INITIAL_METADATA)) { + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_INITIAL_METADATA", oas); if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_chttp2_incoming_metadata_buffer_publish(&oas->s->state.rs.initial_metadata, - stream_op->recv_initial_metadata); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE, NULL); + grpc_chttp2_incoming_metadata_buffer_publish( + &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, + GRPC_ERROR_NONE, NULL); } else { - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, + GRPC_ERROR_CANCELLED, NULL); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; - } else if (stream_op->send_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_MESSAGE)) { + } else if (stream_op->send_message && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_MESSAGE", oas); - // TODO (makdharma): Make into a standalone function gpr_slice_buffer write_slice_buffer; gpr_slice slice; gpr_slice_buffer_init(&write_slice_buffer); grpc_byte_stream_next(NULL, stream_op->send_message, &slice, stream_op->send_message->length, NULL); - // Check that compression flag is not ON. We don't support compression yet. - // TODO (makdharma): add compression support + /* Check that compression flag is OFF. We don't support compression yet. */ GPR_ASSERT(stream_op->send_message->flags == 0); gpr_slice_buffer_add(&write_slice_buffer, slice); - GPR_ASSERT(write_slice_buffer.count == 1); // Empty request not handled yet + GPR_ASSERT(write_slice_buffer.count == + 1); /* Empty request not handled yet */ if (write_slice_buffer.count > 0) { int write_buffer_size; - create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, &write_buffer_size); - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", stream_state->ws.write_buffer); + create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, + &write_buffer_size); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", + stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, - write_buffer_size, false); // TODO: What if this is not the last write? + write_buffer_size, false); result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_MESSAGE] = true; oas->state.state_op_done[OP_SEND_MESSAGE] = true; - } else if (stream_op->recv_message && op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_MESSAGE)) { + } else if (stream_op->recv_message && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.length_field_received == false) { - if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { - // Start a read operation for data + if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && + stream_state->rs.remaining_bytes == 0) { + /* Start a read operation for data */ stream_state->rs.length_field_received = true; stream_state->rs.length_field = stream_state->rs.remaining_bytes = - parse_grpc_header((const uint8_t *)stream_state->rs.read_buffer); - CRONET_LOG(GPR_DEBUG, "length field = %d", stream_state->rs.length_field); + parse_grpc_header((const uint8_t *)stream_state->rs.read_buffer); + CRONET_LOG(GPR_DEBUG, "length field = %d", + stream_state->rs.length_field); if (stream_state->rs.length_field > 0) { - stream_state->rs.read_buffer = gpr_malloc(stream_state->rs.length_field); + stream_state->rs.read_buffer = + gpr_malloc(stream_state->rs.length_field); GPR_ASSERT(stream_state->rs.read_buffer); stream_state->rs.remaining_bytes = stream_state->rs.length_field; stream_state->rs.received_bytes = 0; CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); - stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + stream_state->state_op_done[OP_READ_REQ_MADE] = + true; /* Indicates that at least one read request has been made */ cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, - stream_state->rs.remaining_bytes); + stream_state->rs.remaining_bytes); result = ACTION_TAKEN_WITH_CALLBACK; } else { stream_state->rs.remaining_bytes = 0; CRONET_LOG(GPR_DEBUG, "read operation complete. Empty response."); gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); - grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); - *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, + &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = + (grpc_byte_buffer *)&stream_state->rs.sbs; + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; - oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. + oas->state.state_op_done[OP_RECV_MESSAGE] = true; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_state->rs.remaining_bytes == 0) { - // Start a read operation for first 5 bytes (GRPC header) + /* Start a read operation for first 5 bytes (GRPC header) */ stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; stream_state->rs.received_bytes = 0; CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); - stream_state->state_op_done[OP_READ_REQ_MADE] = true; // If at least one read request has been made + stream_state->state_op_done[OP_READ_REQ_MADE] = + true; /* Indicates that at least one read request has been made */ cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, stream_state->rs.remaining_bytes); } result = ACTION_TAKEN_WITH_CALLBACK; } else if (stream_state->rs.remaining_bytes == 0) { CRONET_LOG(GPR_DEBUG, "read operation complete"); - gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)stream_state->rs.length_field); + gpr_slice read_data_slice = + gpr_slice_malloc((uint32_t)stream_state->rs.length_field); uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice); - memcpy(dst_p, stream_state->rs.read_buffer, (size_t)stream_state->rs.length_field); + memcpy(dst_p, stream_state->rs.read_buffer, + (size_t)stream_state->rs.length_field); gpr_slice_buffer_init(&stream_state->rs.read_slice_buffer); - gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, read_data_slice); - grpc_slice_buffer_stream_init(&stream_state->rs.sbs, &stream_state->rs.read_slice_buffer, 0); - *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); + gpr_slice_buffer_add(&stream_state->rs.read_slice_buffer, + read_data_slice); + grpc_slice_buffer_stream_init(&stream_state->rs.sbs, + &stream_state->rs.read_slice_buffer, 0); + *((grpc_byte_buffer **)stream_op->recv_message) = + (grpc_byte_buffer *)&stream_state->rs.sbs; + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; - oas->state.state_op_done[OP_RECV_MESSAGE] = true; // Also set per op state. - // Clear read state of the stream, so next read op (if it were to come) will work - stream_state->rs.received_bytes = stream_state->rs.remaining_bytes = stream_state->rs.length_field_received = 0; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; + /* Clear read state of the stream, so next read op (if it were to come) + * will work */ + stream_state->rs.received_bytes = stream_state->rs.remaining_bytes = + stream_state->rs.length_field_received = 0; result = ACTION_TAKEN_NO_CALLBACK; } } else if (stream_op->recv_trailing_metadata && - op_can_be_run(stream_op, stream_state, &oas->state, OP_RECV_TRAILING_METADATA)) { + op_can_be_run(stream_op, stream_state, &oas->state, + OP_RECV_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_TRAILING_METADATA", oas); if (oas->s->state.rs.trailing_metadata_valid) { grpc_chttp2_incoming_metadata_buffer_publish( - &oas->s->state.rs.trailing_metadata, stream_op->recv_trailing_metadata); + &oas->s->state.rs.trailing_metadata, + stream_op->recv_trailing_metadata); stream_state->rs.trailing_metadata_valid = false; } stream_state->state_op_done[OP_RECV_TRAILING_METADATA] = true; result = ACTION_TAKEN_NO_CALLBACK; - } else if (stream_op->send_trailing_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_TRAILING_METADATA)) { + } else if (stream_op->send_trailing_metadata && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_SEND_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, "", 0, true); stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; result = ACTION_TAKEN_WITH_CALLBACK; - } else if (stream_op->cancel_error && op_can_be_run(stream_op, stream_state, &oas->state, OP_CANCEL_ERROR)) { + } else if (stream_op->cancel_error && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_CANCEL_ERROR)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_CANCEL_ERROR", oas); CRONET_LOG(GPR_DEBUG, "W: cronet_bidirectional_stream_cancel(%p)", s->cbs); - // Cancel might have come before creation of stream if (s->cbs) { cronet_bidirectional_stream_cancel(s->cbs); } stream_state->state_op_done[OP_CANCEL_ERROR] = true; result = ACTION_TAKEN_WITH_CALLBACK; - } else if (stream_op->on_complete && op_can_be_run(stream_op, stream_state, &oas->state, OP_ON_COMPLETE)) { - // All ops are complete. Call the on_complete callback + } else if (stream_op->on_complete && + op_can_be_run(stream_op, stream_state, &oas->state, + OP_ON_COMPLETE)) { + /* All actions in this stream_op are complete. Call the on_complete callback + */ CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); - //CRONET_LOG(GPR_DEBUG, "calling on_complete"); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, NULL); - // Instead of setting stream state, use the op state as on_complete is on per op basis + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, + NULL); oas->state.state_op_done[OP_ON_COMPLETE] = true; - oas->done = true; // Mark this op as completed - // reset any send or receive message state. + oas->done = true; + /* reset any send or receive message state. */ stream_state->state_callback_received[OP_SEND_MESSAGE] = false; stream_state->state_op_done[OP_SEND_MESSAGE] = false; result = ACTION_TAKEN_NO_CALLBACK; - // If this is the on_complete callback being called for a received message - make a note - if (stream_op->recv_message) stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; + /* If this is the on_complete callback being called for a received message - + make a note */ + if (stream_op->recv_message) + stream_state->state_op_done[OP_RECV_MESSAGE_AND_ON_COMPLETE] = true; } else { - //CRONET_LOG(GPR_DEBUG, "No op ready to run"); result = NO_ACTION_POSSIBLE; } return result; } -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +/* + Functions used by upper layers to access transport functionality. +*/ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, grpc_stream *gs, grpc_stream_refcount *refcount, @@ -720,7 +837,8 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, memset(&s->state.rs, 0, sizeof(s->state.rs)); memset(&s->state.ws, 0, sizeof(s->state.ws)); memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); - memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); + memset(s->state.state_callback_received, 0, + sizeof(s->state.state_callback_received)); gpr_mu_init(&s->mu); s->exec_ctx = *exec_ctx; return 0; @@ -744,19 +862,16 @@ static void perform_stream_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, } static void destroy_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_stream *gs, void *and_free_memory) { -} + grpc_stream *gs, void *and_free_memory) {} -static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { -} +static void destroy_transport(grpc_exec_ctx *exec_ctx, grpc_transport *gt) {} static char *get_peer(grpc_exec_ctx *exec_ctx, grpc_transport *gt) { return NULL; } static void perform_op(grpc_exec_ctx *exec_ctx, grpc_transport *gt, - grpc_transport_op *op) { -} + grpc_transport_op *op) {} const grpc_transport_vtable grpc_cronet_vtable = {sizeof(stream_obj), "cronet_http", From 280885eaa5e48912fe623dcbad8ad398d8eb405d Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Thu, 11 Aug 2016 14:16:36 -0700 Subject: [PATCH 28/43] WIP --- .../cronet/transport/cronet_transport.c | 177 +++++++++++++----- 1 file changed, 128 insertions(+), 49 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 1d756039809..8f11ef73792 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -50,16 +50,13 @@ #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" #define GRPC_HEADER_SIZE_IN_BYTES 5 -// maximum ops in a batch. There is not much thinking behind this limit, except -// that it seems to be enough for most use cases. -#define MAX_PENDING_OPS 100 #define CRONET_LOG(...) \ { \ if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ } -// TODO (makdharma): Hook up into the wider tracing mechanism +/* TODO (makdharma): Hook up into the wider tracing mechanism */ int grpc_cronet_trace = 1; enum OP_RESULT { @@ -68,7 +65,7 @@ enum OP_RESULT { NO_ACTION_POSSIBLE }; -// Used for printing debug +/* Used for printing debug */ const char *op_result_string[] = {"ACTION_TAKEN_WITH_CALLBACK", "ACTION_TAKEN_NO_CALLBACK", "NO_ACTION_POSSIBLE"}; @@ -129,7 +126,7 @@ static cronet_bidirectional_stream_callback cronet_callbacks = { on_failed, on_canceled}; -// Cronet transport object +/* Cronet transport object */ struct grpc_cronet_transport { grpc_transport base; /* must be first element in this structure */ cronet_engine *engine; @@ -146,6 +143,7 @@ struct read_state { int length_field; char grpc_header_bytes[GRPC_HEADER_SIZE_IN_BYTES]; char *payload_field; + bool read_stream_closed; /* vars for holding data destined for the application */ struct grpc_slice_buffer_stream sbs; @@ -177,14 +175,13 @@ struct op_and_state { grpc_transport_stream_op op; struct op_state state; bool done; - struct stream_obj *s; /* Pointer back to the stream object */ + struct stream_obj *s; /* Pointer back to the stream object */ + struct op_and_state *next; /* next op_and_state in the linked list */ }; struct op_storage { - struct op_and_state pending_ops[MAX_PENDING_OPS]; - int wrptr; - int rdptr; int num_pending_ops; + struct op_and_state *head; }; struct stream_obj { @@ -217,20 +214,52 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas); static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { gpr_mu_lock(&s->mu); struct op_storage *storage = &s->storage; - GPR_ASSERT(storage->num_pending_ops < MAX_PENDING_OPS); + /* add new op at the beginning of the linked list. The memory is freed + in remove_from_storage */ + struct op_and_state *new_op = gpr_malloc(sizeof(struct op_and_state)); + memcpy(&new_op->op, op, sizeof(grpc_transport_stream_op)); + memset(&new_op->state, 0, sizeof(new_op->state)); + new_op->s = s; + new_op->done = false; + new_op->next = storage->head; + storage->head = new_op; storage->num_pending_ops++; - CRONET_LOG(GPR_DEBUG, "adding new op @wrptr=%d. %d in the queue.", - storage->wrptr, storage->num_pending_ops); - memcpy(&storage->pending_ops[storage->wrptr].op, op, - sizeof(grpc_transport_stream_op)); - memset(&storage->pending_ops[storage->wrptr].state, 0, - sizeof(storage->pending_ops[storage->wrptr].state)); - storage->pending_ops[storage->wrptr].done = false; - storage->pending_ops[storage->wrptr].s = s; - storage->wrptr = (storage->wrptr + 1) % MAX_PENDING_OPS; + CRONET_LOG(GPR_DEBUG, "adding new op %p. %d in the queue.", new_op, + storage->num_pending_ops); gpr_mu_unlock(&s->mu); } +/* + Traverse the linked list and delete op and free memory +*/ +static void remove_from_storage(struct stream_obj *s, + struct op_and_state *oas) { + struct op_and_state *curr; + if (s->storage.head == NULL || oas == NULL) { + return; + } + if (s->storage.head == oas) { + s->storage.head = oas->next; + gpr_free(oas); + s->storage.num_pending_ops--; + CRONET_LOG(GPR_DEBUG, "Freed %p. Now %d in the queue", oas, + s->storage.num_pending_ops); + } else { + for (curr = s->storage.head; curr != NULL; curr = curr->next) { + if (curr->next == oas) { + curr->next = oas->next; + s->storage.num_pending_ops--; + CRONET_LOG(GPR_DEBUG, "Freed %p. Now %d in the queue", oas, + s->storage.num_pending_ops); + gpr_free(oas); + break; + } else if (curr->next == NULL) { + CRONET_LOG(GPR_ERROR, "Reached end of LL and did not find op to free"); + } + } + } +} + /* Cycle through ops and try to take next action. Break when either an action with callback is taken, or no action is possible. @@ -239,18 +268,21 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { */ static void execute_from_storage(stream_obj *s) { gpr_mu_lock(&s->mu); - for (int i = 0; i < s->storage.wrptr;) { - CRONET_LOG(GPR_DEBUG, "calling execute_stream_op[%d]. done = %d", i, - s->storage.pending_ops[i].done); - if (s->storage.pending_ops[i].done) { - i++; - continue; + for (struct op_and_state *curr = s->storage.head; curr != NULL;) { + CRONET_LOG(GPR_DEBUG, "calling op at %p. done = %d", curr, curr->done); + GPR_ASSERT(curr->done == 0); + enum OP_RESULT result = execute_stream_op(curr); + CRONET_LOG(GPR_DEBUG, "execute_stream_op[%p] returns %s", curr, + op_result_string[result]); + /* if this op is done, then remove it and free memory */ + if (curr->done) { + struct op_and_state *next = curr->next; + remove_from_storage(s, curr); + curr = next; } - enum OP_RESULT result = execute_stream_op(&s->storage.pending_ops[i]); - CRONET_LOG(GPR_DEBUG, "%s = execute_stream_op[%d]", - op_result_string[result], i); + /* continue processing the same op if ACTION_TAKEN_WITHOUT_CALLBACK */ if (result == NO_ACTION_POSSIBLE) { - i++; + curr = curr->next; } else if (result == ACTION_TAKEN_WITH_CALLBACK) { break; } @@ -268,6 +300,14 @@ static void on_failed(cronet_bidirectional_stream *stream, int net_error) { cronet_bidirectional_stream_destroy(s->cbs); s->state.state_callback_received[OP_FAILED] = true; s->cbs = NULL; + if (s->header_array.headers) { + gpr_free(s->header_array.headers); + s->header_array.headers = NULL; + } + if (s->state.ws.write_buffer) { + gpr_free(s->state.ws.write_buffer); + s->state.ws.write_buffer = NULL; + } execute_from_storage(s); } @@ -280,6 +320,14 @@ static void on_canceled(cronet_bidirectional_stream *stream) { cronet_bidirectional_stream_destroy(s->cbs); s->state.state_callback_received[OP_CANCELED] = true; s->cbs = NULL; + if (s->header_array.headers) { + gpr_free(s->header_array.headers); + s->header_array.headers = NULL; + } + if (s->state.ws.write_buffer) { + gpr_free(s->state.ws.write_buffer); + s->state.ws.write_buffer = NULL; + } execute_from_storage(s); } @@ -306,6 +354,7 @@ static void on_request_headers_sent(cronet_bidirectional_stream *stream) { /* Free the memory allocated for headers */ if (s->header_array.headers) { gpr_free(s->header_array.headers); + s->header_array.headers = NULL; } execute_from_storage(s); } @@ -358,6 +407,7 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, stream_obj *s = (stream_obj *)stream->annotation; CRONET_LOG(GPR_DEBUG, "R: on_read_completed(%p, %p, %d)", stream, data, count); + s->state.state_callback_received[OP_RECV_MESSAGE] = true; if (count > 0) { s->state.rs.received_bytes += count; s->state.rs.remaining_bytes -= count; @@ -370,7 +420,9 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, } else { execute_from_storage(s); } - s->state.state_callback_received[OP_RECV_MESSAGE] = true; + } else { + s->state.rs.read_stream_closed = true; + execute_from_storage(s); } } @@ -570,38 +622,51 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, } else if (op_id == OP_ON_COMPLETE) { /* already executed (note we're checking op specific state, not stream state) */ - if (op_state->state_op_done[OP_ON_COMPLETE]) result = false; + if (op_state->state_op_done[OP_ON_COMPLETE]) { + CRONET_LOG(GPR_DEBUG, "Because"); + result = false; + } /* Check if every op that was asked for is done. */ else if (curr_op->send_initial_metadata && - !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) + !stream_state->state_callback_received[OP_SEND_INITIAL_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->send_message && - !stream_state->state_op_done[OP_SEND_MESSAGE]) + } else if (curr_op->send_message && + !op_state->state_op_done[OP_SEND_MESSAGE]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->send_message && - !stream_state->state_callback_received[OP_SEND_MESSAGE]) + } else if (curr_op->send_message && + !stream_state->state_callback_received[OP_SEND_MESSAGE]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->send_trailing_metadata && - !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) + } else if (curr_op->send_trailing_metadata && + !stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->recv_initial_metadata && - !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) + } else if (curr_op->recv_initial_metadata && + !stream_state->state_op_done[OP_RECV_INITIAL_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->recv_message && - !stream_state->state_op_done[OP_RECV_MESSAGE]) + } else if (curr_op->recv_message && + !stream_state->state_op_done[OP_RECV_MESSAGE]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; - else if (curr_op->recv_trailing_metadata) { + } else if (curr_op->recv_trailing_metadata) { /* We aren't done with trailing metadata yet */ - if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) + if (!stream_state->state_op_done[OP_RECV_TRAILING_METADATA]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; + } /* We've asked for actual message in an earlier op, and it hasn't been delivered yet. */ else if (stream_state->state_op_done[OP_READ_REQ_MADE]) { /* If this op is not the one asking for read, (which means some earlier op has asked), and the read hasn't been delivered. */ if (!curr_op->recv_message && - !stream_state->state_op_done[OP_SUCCEEDED]) + !stream_state->state_callback_received[OP_SUCCEEDED]) { + CRONET_LOG(GPR_DEBUG, "Because"); result = false; + } } } /* We should see at least one on_write_completed for the trailers that we @@ -625,6 +690,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { OP_SEND_INITIAL_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_INITIAL_METADATA", oas); /* This OP is the beginning. Reset various states */ + memset(&s->header_array, 0, sizeof(s->header_array)); memset(&stream_state->rs, 0, sizeof(stream_state->rs)); memset(&stream_state->ws, 0, sizeof(stream_state->ws)); memset(stream_state->state_op_done, 0, sizeof(stream_state->state_op_done)); @@ -637,6 +703,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); char *url; + s->header_array.headers = NULL; convert_metadata_to_cronet_headers( stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, &s->header_array.headers, &s->header_array.count); @@ -696,6 +763,13 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; + } else if (stream_state->rs.read_stream_closed == true) { + /* No more data will be received */ + CRONET_LOG(GPR_DEBUG, "read stream closed"); + grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + GRPC_ERROR_NONE, NULL); + stream_state->state_op_done[OP_RECV_MESSAGE] = true; + oas->state.state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.length_field_received == false) { if (stream_state->rs.received_bytes == GRPC_HEADER_SIZE_IN_BYTES && stream_state->rs.remaining_bytes == 0) { @@ -808,9 +882,12 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { NULL); oas->state.state_op_done[OP_ON_COMPLETE] = true; oas->done = true; - /* reset any send or receive message state. */ - stream_state->state_callback_received[OP_SEND_MESSAGE] = false; - stream_state->state_op_done[OP_SEND_MESSAGE] = false; + /* reset any send message state, only if this ON_COMPLETE is about a send. + */ + if (stream_op->send_message) { + stream_state->state_callback_received[OP_SEND_MESSAGE] = false; + stream_state->state_op_done[OP_SEND_MESSAGE] = false; + } result = ACTION_TAKEN_NO_CALLBACK; /* If this is the on_complete callback being called for a received message - make a note */ @@ -831,9 +908,11 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, const void *server_data) { stream_obj *s = (stream_obj *)gs; memset(&s->storage, 0, sizeof(s->storage)); + s->storage.head = NULL; memset(&s->state, 0, sizeof(s->state)); s->curr_op = NULL; s->cbs = NULL; + memset(&s->header_array, 0, sizeof(s->header_array)); memset(&s->state.rs, 0, sizeof(s->state.rs)); memset(&s->state.ws, 0, sizeof(s->state.ws)); memset(s->state.state_op_done, 0, sizeof(s->state.state_op_done)); From 91d519532a79e7309e0d63a246c064398eec5288 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Thu, 11 Aug 2016 15:30:52 -0700 Subject: [PATCH 29/43] removed file from commit --- .../CoreCronetEnd2EndTests.m | 403 ------------------ 1 file changed, 403 deletions(-) delete mode 100644 src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m diff --git a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m b/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m deleted file mode 100644 index d2181120e93..00000000000 --- a/src/objective-c/tests/CoreCronetEnd2EndTests/CoreCronetEnd2EndTests.m +++ /dev/null @@ -1,403 +0,0 @@ -/* - * - * Copyright 2015, Google Inc. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - */ - -/* - * This test file is derived from fixture h2_ssl.c in core end2end test - * (test/core/end2end/fixture/h2_ssl.c). The structure of the fixture is - * preserved as much as possible - * - * This fixture creates a server full stack using chttp2 and a client - * full stack using Cronet. End-to-end tests are run against this - * configuration - * - */ - - -#import -#include "test/core/end2end/end2end_tests.h" - -#include -#include - -#include -#include -#include - -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/security/credentials/credentials.h" -#include "src/core/lib/support/env.h" -#include "src/core/lib/support/string.h" -#include "src/core/lib/support/tmpfile.h" -#include "test/core/end2end/data/ssl_test_data.h" -#include "test/core/util/port.h" -#include "test/core/util/test_config.h" - -#include -#import - -typedef struct fullstack_secure_fixture_data { - char *localaddr; -} fullstack_secure_fixture_data; - -static grpc_end2end_test_fixture chttp2_create_fixture_secure_fullstack( - grpc_channel_args *client_args, grpc_channel_args *server_args) { - grpc_end2end_test_fixture f; - int port = grpc_pick_unused_port_or_die(); - fullstack_secure_fixture_data *ffd = - gpr_malloc(sizeof(fullstack_secure_fixture_data)); - memset(&f, 0, sizeof(f)); - - gpr_join_host_port(&ffd->localaddr, "127.0.0.1", port); - - f.fixture_data = ffd; - f.cq = grpc_completion_queue_create(NULL); - - return f; -} - -static void process_auth_failure(void *state, grpc_auth_context *ctx, - const grpc_metadata *md, size_t md_count, - grpc_process_auth_metadata_done_cb cb, - void *user_data) { - GPR_ASSERT(state == NULL); - cb(user_data, NULL, 0, NULL, 0, GRPC_STATUS_UNAUTHENTICATED, NULL); -} - -static void cronet_init_client_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *client_args, - cronet_engine *cronetEngine) { - fullstack_secure_fixture_data *ffd = f->fixture_data; - f->client = - grpc_cronet_secure_channel_create(cronetEngine, ffd->localaddr, client_args, NULL); - GPR_ASSERT(f->client != NULL); -} - -static void chttp2_init_server_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *server_args, - grpc_server_credentials *server_creds) { - fullstack_secure_fixture_data *ffd = f->fixture_data; - if (f->server) { - grpc_server_destroy(f->server); - } - f->server = grpc_server_create(server_args, NULL); - grpc_server_register_completion_queue(f->server, f->cq, NULL); - GPR_ASSERT(grpc_server_add_secure_http2_port(f->server, ffd->localaddr, - server_creds)); - grpc_server_credentials_release(server_creds); - grpc_server_start(f->server); -} - -static void chttp2_tear_down_secure_fullstack(grpc_end2end_test_fixture *f) { - fullstack_secure_fixture_data *ffd = f->fixture_data; - gpr_free(ffd->localaddr); - gpr_free(ffd); -} - -static void cronet_init_client_simple_ssl_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *client_args) { - grpc_arg ssl_name_override = {GRPC_ARG_STRING, - GRPC_SSL_TARGET_NAME_OVERRIDE_ARG, - {"foo.test.google.fr"}}; - - grpc_channel_args *new_client_args = - grpc_channel_args_copy_and_add(client_args, &ssl_name_override, 1); - static bool done = false; - // TODO (makdharma): DO NOT CHECK IN THIS HACK!!! - if (!done) { - done = true; - [Cronet setHttp2Enabled:YES]; - NSURL *url = [[[NSFileManager defaultManager] - URLsForDirectory:NSDocumentDirectory inDomains:NSUserDomainMask] lastObject]; - NSLog(@"Documents directory: %@", url); - [Cronet start]; - [Cronet startNetLogToFile: @"Documents/cronet_netlog.json" logBytes:YES]; - } - cronet_engine *cronetEngine = [Cronet getGlobalEngine]; - - cronet_init_client_secure_fullstack(f, new_client_args, cronetEngine); - grpc_channel_args_destroy(new_client_args); -} - -static int fail_server_auth_check(grpc_channel_args *server_args) { - size_t i; - if (server_args == NULL) return 0; - for (i = 0; i < server_args->num_args; i++) { - if (strcmp(server_args->args[i].key, FAIL_AUTH_CHECK_SERVER_ARG_NAME) == - 0) { - return 1; - } - } - return 0; -} - -static void chttp2_init_server_simple_ssl_secure_fullstack( - grpc_end2end_test_fixture *f, grpc_channel_args *server_args) { - grpc_ssl_pem_key_cert_pair pem_cert_key_pair = {test_server1_key, - test_server1_cert}; - grpc_server_credentials *ssl_creds = - grpc_ssl_server_credentials_create(NULL, &pem_cert_key_pair, 1, 0, NULL); - if (fail_server_auth_check(server_args)) { - grpc_auth_metadata_processor processor = {process_auth_failure, NULL, NULL}; - grpc_server_credentials_set_auth_metadata_processor(ssl_creds, processor); - } - chttp2_init_server_secure_fullstack(f, server_args, ssl_creds); -} - -/* All test configurations */ - -static grpc_end2end_test_config configs[] = { - {"chttp2/simple_ssl_fullstack", - FEATURE_MASK_SUPPORTS_DELAYED_CONNECTION | - FEATURE_MASK_SUPPORTS_PER_CALL_CREDENTIALS, - chttp2_create_fixture_secure_fullstack, - cronet_init_client_simple_ssl_secure_fullstack, - chttp2_init_server_simple_ssl_secure_fullstack, - chttp2_tear_down_secure_fullstack}, -}; - - - -static char *roots_filename; - -@interface CoreCronetEnd2EndTests : XCTestCase - -@end - -@implementation CoreCronetEnd2EndTests - - -// The setUp() function is run before the test cases run and only run once -+ (void)setUp { - [super setUp]; - - FILE *roots_file; - size_t roots_size = strlen(test_root_cert); - - char *argv[] = {"CoreCronetEnd2EndTests"}; - grpc_test_init(1, argv); - grpc_end2end_tests_pre_init(); - - /* Set the SSL roots env var. */ - roots_file = gpr_tmpfile("chttp2_simple_ssl_fullstack_test", &roots_filename); - GPR_ASSERT(roots_filename != NULL); - GPR_ASSERT(roots_file != NULL); - GPR_ASSERT(fwrite(test_root_cert, 1, roots_size, roots_file) == roots_size); - fclose(roots_file); - gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_filename); - - grpc_init(); - -} - -// The tearDown() function is run after all test cases finish running -+ (void)tearDown { - grpc_shutdown(); - - /* Cleanup. */ - remove(roots_filename); - gpr_free(roots_filename); - - [super tearDown]; -} - -- (void)testIndividualCase:(char*)test_case { - char *argv[] = {"h2_ssl", test_case}; - - for (int i = 0; i < sizeof(configs) / sizeof(*configs); i++) { - grpc_end2end_tests(sizeof(argv) / sizeof(argv[0]), argv, configs[i]); - } -} - -// TODO(mxyan): Use NSStringFromSelector(_cmd) to acquire test name from the -// test case method name, so that bodies of test cases can stay identical -- (void)testBadHostname { - [self testIndividualCase:"bad_hostname"]; -} - -- (void)testBinaryMetadata { - //[self testIndividualCase:"binary_metadata"]; -} - -- (void)testCallCreds { - [self testIndividualCase:"call_creds"]; -} - -- (void)testCancelAfterAccept { - [self testIndividualCase:"cancel_after_accept"]; -} - -- (void)testCancelAfterClientDone { - [self testIndividualCase:"cancel_after_client_done"]; -} - -- (void)testCancelAfterInvoke { - [self testIndividualCase:"cancel_after_invoke"]; -} - -- (void)testCancelBeforeInvoke { - [self testIndividualCase:"cancel_before_invoke"]; -} - -- (void)testCancelInAVacuum { - [self testIndividualCase:"cancel_in_a_vacuum"]; -} - -- (void)testCancelWithStatus { - [self testIndividualCase:"cancel_with_status"]; -} - -- (void)testCompressedPayload { - [self testIndividualCase:"compressed_payload"]; -} - -- (void)testConnectivity { - [self testIndividualCase:"connectivity"]; -} - -- (void)testDefaultHost { - [self testIndividualCase:"default_host"]; -} - -- (void)testDisappearingServer { - [self testIndividualCase:"disappearing_server"]; -} - -- (void)testEmptyBatch { - [self testIndividualCase:"empty_batch"]; -} - -- (void)testFilterCausesClose { - [self testIndividualCase:"filter_causes_close"]; -} - -- (void)testGracefulServerShutdown { - [self testIndividualCase:"graceful_server_shutdown"]; -} - -- (void)testHighInitialSeqno { - [self testIndividualCase:"high_initial_seqno"]; -} - -- (void)testHpackSize { - [self testIndividualCase:"hpack_size"]; -} - -- (void)testIdempotentRequest { - [self testIndividualCase:"idempotent_request"]; -} - -- (void)testInvokeLargeRequest { - [self testIndividualCase:"invoke_large_request"]; -} - -- (void)testLargeMetadata { - [self testIndividualCase:"large_metadata"]; -} - -- (void)testMaxConcurrentStreams { - [self testIndividualCase:"max_concurrent_streams"]; -} - -- (void)testMaxMessageLength { - [self testIndividualCase:"max_message_length"]; -} - -- (void)testNegativeDeadline { - [self testIndividualCase:"negative_deadline"]; -} - -- (void)testNetworkStatusChange { - [self testIndividualCase:"network_status_change"]; -} - -- (void)testNoOp { - [self testIndividualCase:"no_op"]; -} - -- (void)testPayload { - [self testIndividualCase:"payload"]; -} - -- (void)testPing { - [self testIndividualCase:"ping"]; -} - -- (void)testPingPongStreaming { - [self testIndividualCase:"ping_pong_streaming"]; -} - -- (void)testRegisteredCall { - [self testIndividualCase:"registered_call"]; -} - -- (void)testRequestWithFlags { - [self testIndividualCase:"request_with_flags"]; -} - -- (void)testRequestWithPayload { - [self testIndividualCase:"request_with_payload"]; -} - -- (void)testServerFinishesRequest { - [self testIndividualCase:"server_finishes_request"]; -} - -- (void)testShutdownFinishesCalls { - [self testIndividualCase:"shutdown_finishes_calls"]; -} - -- (void)testShutdownFinishesTags { - [self testIndividualCase:"shutdown_finishes_tags"]; -} - -- (void)testSimpleDelayedRequest { - [self testIndividualCase:"simple_delayed_request"]; -} - -- (void)testSimpleMetadata { - [self testIndividualCase:"simple_metadata"]; -} - -- (void)testSimpleRequest { - [self testIndividualCase:"simple_request"]; -} - -- (void)testStreamingErrorResponse { - [self testIndividualCase:"streaming_error_response"]; -} - -- (void)testTrailingMetadata { - [self testIndividualCase:"trailing_metadata"]; -} - -@end From 4272cac7aeb577a0f523f1e69f7dddacd79d3a66 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Fri, 12 Aug 2016 15:25:46 -0700 Subject: [PATCH 30/43] Rewrite ProtoFileParser --- test/cpp/util/grpc_tool.cc | 144 +++++++++++----------- test/cpp/util/proto_file_parser.cc | 187 ++++++++++++++++++++--------- test/cpp/util/proto_file_parser.h | 44 ++++--- 3 files changed, 235 insertions(+), 140 deletions(-) diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index e227e6027dc..a181ee7597b 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -31,7 +31,7 @@ * */ -#include "test/cpp/util/grpc_tool.h" +#include "grpc_tool.h" #include #include @@ -55,15 +55,14 @@ DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); DEFINE_bool(use_auth, false, "Whether to create default google credentials."); -DEFINE_string(input_binary_file, "", - "Path to input file containing serialized request."); -DEFINE_string(output_binary_file, "", - "Path to output file to write serialized response."); +DEFINE_bool(remotedb, true, "Use server types to parse and format messages"); DEFINE_string(metadata, "", "Metadata to send to server, in the form of key1:val1:key2:val2"); DEFINE_string(proto_path, ".", "Path to look for the proto file."); -// TODO(zyc): support a list of input proto files -DEFINE_string(protofiles, "", "Name of the proto file."); +DEFINE_string(proto_file, "", "Name of the proto file."); +DEFINE_bool(binary_input, false, "Input in binary format"); +DEFINE_bool(binary_output, false, "Output in binary format"); +DEFINE_string(infile, "", "Input file (default is stdin)"); namespace grpc { namespace testing { @@ -73,8 +72,22 @@ class GrpcTool { public: explicit GrpcTool(); virtual ~GrpcTool() {} + bool Help(int argc, const char** argv, GrpcToolOutputCallback callback); bool CallMethod(int argc, const char** argv, GrpcToolOutputCallback callback); + // TODO(zyc): implement the following methods + // bool ListServices(int argc, const char** argv, GrpcToolOutputCallback + // callback); + // bool PrintType(int argc, const char** argv, GrpcToolOutputCallback + // callback); + // bool PrintTypeId(int argc, const char** argv, GrpcToolOutputCallback + // callback); + // bool ParseMessage(int argc, const char** argv, GrpcToolOutputCallback + // callback); + // bool ToText(int argc, const char** argv, GrpcToolOutputCallback callback); + // bool ToBinary(int argc, const char** argv, GrpcToolOutputCallback + // callback); + void SetPrintCommandMode(int exit_status) { print_command_usage_ = true; usage_exit_status_ = exit_status; @@ -82,6 +95,7 @@ class GrpcTool { private: void CommandUsage(const grpc::string& usage) const; + std::shared_ptr NewChannel(const grpc::string& server_address); bool print_command_usage_; int usage_exit_status_; }; @@ -222,6 +236,21 @@ void GrpcTool::CommandUsage(const grpc::string& usage) const { } } +std::shared_ptr GrpcTool::NewChannel( + const grpc::string& server_address) { + std::shared_ptr creds; + if (!FLAGS_enable_ssl) { + creds = grpc::InsecureChannelCredentials(); + } else { + if (FLAGS_use_auth) { + creds = grpc::GoogleDefaultCredentials(); + } else { + creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); + } + } + return grpc::CreateChannel(server_address, creds); +} + bool GrpcTool::Help(int argc, const char** argv, GrpcToolOutputCallback callback) { CommandUsage( @@ -250,17 +279,18 @@ bool GrpcTool::CallMethod(int argc, const char** argv, " ; Exported service name\n" " ; Method name\n" " ; Text protobuffer (overrides infile)\n" - " --protofiles ; Comma separated proto files used as a" + " --proto_file ; Comma separated proto files used as a" " fallback when parsing request/response\n" " --proto_path ; The search path of proto files, valid" - " only when --protofiles is given\n" + " only when --proto_file is given\n" " --metadata ; The metadata to be sent to the server\n" " --enable_ssl ; Set whether to use tls\n" " --use_auth ; Set whether to create default google" " credentials\n" + " --infile ; Input filename (defaults to stdin)\n" " --outfile ; Output filename (defaults to stdout)\n" - " --input_binary_file ; Path to input file in binary format\n" - " --binary_output ; Path to output file in binary format\n"); + " --binary_input ; Input in binary format\n" + " --binary_output ; Output in binary format\n"); std::stringstream output_ss; grpc::string request_text; @@ -271,63 +301,44 @@ bool GrpcTool::CallMethod(int argc, const char** argv, if (argc == 3) { request_text = argv[2]; - } - - std::shared_ptr creds; - if (!FLAGS_enable_ssl) { - creds = grpc::InsecureChannelCredentials(); + if (!FLAGS_infile.empty()) { + fprintf(stderr, "warning: request given in argv, ignoring --infile\n"); + } } else { - if (FLAGS_use_auth) { - creds = grpc::GoogleDefaultCredentials(); + std::stringstream input_stream; + if (FLAGS_infile.empty()) { + if (isatty(STDIN_FILENO)) { + fprintf(stderr, "reading request message from stdin...\n"); + } + input_stream << std::cin.rdbuf(); } else { - creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); - } - } - std::shared_ptr channel = - grpc::CreateChannel(server_address, creds); - - if (request_text.empty() && FLAGS_input_binary_file.empty()) { - if (isatty(STDIN_FILENO)) { - std::cout << "reading request message from stdin..." << std::endl; + std::ifstream input_file(FLAGS_infile, std::ios::in | std::ios::binary); + input_stream << input_file.rdbuf(); + input_file.close(); } - std::stringstream input_stream; - input_stream << std::cin.rdbuf(); request_text = input_stream.str(); } - if (!request_text.empty()) { - if (!FLAGS_protofiles.empty()) { - parser.reset(new grpc::testing::ProtoFileParser( - FLAGS_proto_path, FLAGS_protofiles, method_name)); - } else { - parser.reset(new grpc::testing::ProtoFileParser(channel, method_name)); - } - method_name = parser->GetFullMethodName(); + std::shared_ptr channel = NewChannel(server_address); + if (!FLAGS_binary_input || !FLAGS_binary_output) { + parser.reset( + new grpc::testing::ProtoFileParser(FLAGS_remotedb ? channel : nullptr, + FLAGS_proto_path, FLAGS_proto_file)); if (parser->HasError()) { - return 1; - } - - if (!FLAGS_input_binary_file.empty()) { - std::cout - << "warning: request given in argv, ignoring --input_binary_file" - << std::endl; + return false; } } - if (parser) { - serialized_request_proto = - parser->GetSerializedProto(request_text, true /* is_request */); + if (FLAGS_binary_input) { + serialized_request_proto = request_text; + } else { + serialized_request_proto = parser->GetSerializedProtoFromMethod( + method_name, request_text, true /* is_request */); if (parser->HasError()) { - return 1; + return false; } - } else if (!FLAGS_input_binary_file.empty()) { - std::ifstream input_file(FLAGS_input_binary_file, - std::ios::in | std::ios::binary); - std::stringstream input_stream; - input_stream << input_file.rdbuf(); - serialized_request_proto = input_stream.str(); } - std::cout << "connecting to " << server_address << std::endl; + std::cerr << "connecting to " << server_address << std::endl; grpc::string serialized_response_proto; std::multimap client_metadata; @@ -336,30 +347,27 @@ bool GrpcTool::CallMethod(int argc, const char** argv, ParseMetadataFlag(&client_metadata); PrintMetadata(client_metadata, "Sending client initial metadata:"); grpc::Status s = grpc::testing::CliCall::Call( - channel, method_name, serialized_request_proto, - &serialized_response_proto, client_metadata, &server_initial_metadata, - &server_trailing_metadata); + channel, parser->GetFormatedMethodName(method_name), + serialized_request_proto, &serialized_response_proto, client_metadata, + &server_initial_metadata, &server_trailing_metadata); PrintMetadata(server_initial_metadata, "Received initial metadata from server:"); PrintMetadata(server_trailing_metadata, "Received trailing metadata from server:"); if (s.ok()) { - std::cout << "Rpc succeeded with OK status" << std::endl; - if (parser) { - grpc::string response_text = parser->GetTextFormat( - serialized_response_proto, false /* is_request */); + std::cerr << "Rpc succeeded with OK status" << std::endl; + if (FLAGS_binary_output) { + output_ss << serialized_response_proto; + } else { + grpc::string response_text = parser->GetTextFormatFromMethod( + method_name, serialized_response_proto, false /* is_request */); if (parser->HasError()) { return false; } output_ss << "Response: \n " << response_text << std::endl; } - if (!FLAGS_output_binary_file.empty()) { - std::ofstream output_file(FLAGS_output_binary_file, - std::ios::trunc | std::ios::binary); - output_file << serialized_response_proto; - } } else { - std::cout << "Rpc failed with status code " << s.error_code() + std::cerr << "Rpc failed with status code " << s.error_code() << ", error message: " << s.error_message() << std::endl; } diff --git a/test/cpp/util/proto_file_parser.cc b/test/cpp/util/proto_file_parser.cc index 5b0d925e1c3..f7ad17e809c 100644 --- a/test/cpp/util/proto_file_parser.cc +++ b/test/cpp/util/proto_file_parser.cc @@ -71,7 +71,7 @@ class ErrorPrinter void AddWarning(const grpc::string& filename, int line, int column, const grpc::string& message) GRPC_OVERRIDE { - std::cout << "warning " << filename << " " << line << " " << column << " " + std::cerr << "warning " << filename << " " << line << " " << column << " " << message << std::endl; } @@ -79,62 +79,72 @@ class ErrorPrinter ProtoFileParser* parser_; // not owned }; -ProtoFileParser::ProtoFileParser(const grpc::string& proto_path, - const grpc::string& file_name, - const grpc::string& method) +ProtoFileParser::ProtoFileParser(std::shared_ptr channel, + const grpc::string& proto_path, + const grpc::string& protofiles) : has_error_(false) { - source_tree_.MapPath("", proto_path); - error_printer_.reset(new ErrorPrinter(this)); - importer_.reset(new google::protobuf::compiler::Importer( - &source_tree_, error_printer_.get())); - const auto* file_desc = importer_->Import(file_name); - if (!file_desc) { - LogError(""); - return; + std::vector service_list; + if (channel) { + reflection_db_.reset(new grpc::ProtoReflectionDescriptorDatabase(channel)); + reflection_db_->GetServices(&service_list); } - dynamic_factory_.reset( - new google::protobuf::DynamicMessageFactory(importer_->pool())); - std::vector service_desc_list; - for (int i = 0; i < file_desc->service_count(); i++) { - service_desc_list.push_back(file_desc->service(i)); - } - InitProtoFileParser(method, service_desc_list); -} + if (!protofiles.empty()) { + source_tree_.MapPath("", proto_path); + error_printer_.reset(new ErrorPrinter(this)); + importer_.reset(new google::protobuf::compiler::Importer( + &source_tree_, error_printer_.get())); -ProtoFileParser::ProtoFileParser(std::shared_ptr channel, - const grpc::string& method) - : has_error_(false), - desc_db_(new grpc::ProtoReflectionDescriptorDatabase(channel)), - desc_pool_(new google::protobuf::DescriptorPool(desc_db_.get())) { - std::vector service_list; - if (!desc_db_->GetServices(&service_list)) { - LogError( - "Failed to get services from the server, " - "it may not have the reflection service.\n" - "Please try to use the --protofiles option to provide a proto file."); + grpc::string file_name; + std::stringstream ss(protofiles); + while (std::getline(ss, file_name, ',')) { + std::cerr << file_name << std::endl; + const auto* file_desc = importer_->Import(file_name); + if (file_desc) { + for (int i = 0; i < file_desc->service_count(); i++) { + service_desc_list_.push_back(file_desc->service(i)); + } + } else { + std::cerr << file_name << " not found" << std::endl; + } + } + + file_db_.reset( + new google::protobuf::DescriptorPoolDatabase(*importer_->pool())); } - if (has_error_) { + + if (!reflection_db_ && !file_db_) { + LogError("No available proto database"); return; } + + if (!reflection_db_) { + desc_db_ = std::move(file_db_); + } else if (!file_db_) { + desc_db_ = std::move(reflection_db_); + } else { + desc_db_.reset(new google::protobuf::MergedDescriptorDatabase( + reflection_db_.get(), file_db_.get())); + } + + desc_pool_.reset(new google::protobuf::DescriptorPool(desc_db_.get())); dynamic_factory_.reset( new google::protobuf::DynamicMessageFactory(desc_pool_.get())); - std::vector service_desc_list; for (auto it = service_list.begin(); it != service_list.end(); it++) { - service_desc_list.push_back(desc_pool_->FindServiceByName(*it)); + if (const google::protobuf::ServiceDescriptor* service_desc = + desc_pool_->FindServiceByName(*it)) { + service_desc_list_.push_back(service_desc); + } } - InitProtoFileParser(method, service_desc_list); } ProtoFileParser::~ProtoFileParser() {} -void ProtoFileParser::InitProtoFileParser( - const grpc::string& method, - const std::vector - service_desc_list) { +grpc::string ProtoFileParser::GetFullMethodName(const grpc::string& method) { + has_error_ = false; const google::protobuf::MethodDescriptor* method_descriptor = nullptr; - for (auto it = service_desc_list.begin(); it != service_desc_list.end(); + for (auto it = service_desc_list_.begin(); it != service_desc_list_.end(); it++) { const auto* service_desc = *it; for (int j = 0; j < service_desc->method_count(); j++) { @@ -154,26 +164,80 @@ void ProtoFileParser::InitProtoFileParser( LogError("Method name not found"); } if (has_error_) { - return; + return ""; } - full_method_name_ = method_descriptor->full_name(); - size_t last_dot = full_method_name_.find_last_of('.'); + + return method_descriptor->full_name(); +} + +grpc::string ProtoFileParser::GetFormatedMethodName( + const grpc::string& method) { + has_error_ = false; + grpc::string formated_method_name = GetFullMethodName(method); + if (has_error_) { + return ""; + } + size_t last_dot = formated_method_name.find_last_of('.'); if (last_dot != grpc::string::npos) { - full_method_name_[last_dot] = '/'; + formated_method_name[last_dot] = '/'; + } + formated_method_name.insert(formated_method_name.begin(), '/'); + return formated_method_name; +} + +grpc::string ProtoFileParser::GetMessageTypeFromMethod( + const grpc::string& method, bool is_request) { + has_error_ = false; + grpc::string full_method_name = GetFullMethodName(method); + if (has_error_) { + return ""; + } + const google::protobuf::MethodDescriptor* method_desc = + desc_pool_->FindMethodByName(full_method_name); + if (!method_desc) { + LogError("Method not found"); + return ""; } - full_method_name_.insert(full_method_name_.begin(), '/'); - request_prototype_.reset( - dynamic_factory_->GetPrototype(method_descriptor->input_type())->New()); - response_prototype_.reset( - dynamic_factory_->GetPrototype(method_descriptor->output_type())->New()); + return is_request ? method_desc->input_type()->full_name() + : method_desc->output_type()->full_name(); } -grpc::string ProtoFileParser::GetSerializedProto( - const grpc::string& text_format_proto, bool is_request) { +grpc::string ProtoFileParser::GetSerializedProtoFromMethod( + const grpc::string& method, const grpc::string& text_format_proto, + bool is_request) { + has_error_ = false; + grpc::string message_type_name = GetMessageTypeFromMethod(method, is_request); + if (has_error_) { + return ""; + } + return GetSerializedProtoFromMessageType(message_type_name, + text_format_proto); +} + +grpc::string ProtoFileParser::GetTextFormatFromMethod( + const grpc::string& method, const grpc::string& serialized_proto, + bool is_request) { + has_error_ = false; + grpc::string message_type_name = GetMessageTypeFromMethod(method, is_request); + if (has_error_) { + return ""; + } + return GetTextFormatFromMessageType(message_type_name, serialized_proto); +} + +grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( + const grpc::string& message_type_name, + const grpc::string& text_format_proto) { + has_error_ = false; grpc::string serialized; - grpc::protobuf::Message* msg = - is_request ? request_prototype_.get() : response_prototype_.get(); + const google::protobuf::Descriptor* desc = + desc_pool_->FindMessageTypeByName(message_type_name); + if (!desc) { + LogError("Message type not found"); + return ""; + } + grpc::protobuf::Message* msg = dynamic_factory_->GetPrototype(desc)->New(); bool ok = google::protobuf::TextFormat::ParseFromString(text_format_proto, msg); if (!ok) { @@ -188,10 +252,17 @@ grpc::string ProtoFileParser::GetSerializedProto( return serialized; } -grpc::string ProtoFileParser::GetTextFormat( - const grpc::string& serialized_proto, bool is_request) { - grpc::protobuf::Message* msg = - is_request ? request_prototype_.get() : response_prototype_.get(); +grpc::string ProtoFileParser::GetTextFormatFromMessageType( + const grpc::string& message_type_name, + const grpc::string& serialized_proto) { + has_error_ = false; + const google::protobuf::Descriptor* desc = + desc_pool_->FindMessageTypeByName(message_type_name); + if (!desc) { + LogError("Message type not found"); + return ""; + } + grpc::protobuf::Message* msg = dynamic_factory_->GetPrototype(desc)->New(); if (!msg->ParseFromString(serialized_proto)) { LogError("Failed to deserialize proto."); return ""; @@ -206,7 +277,7 @@ grpc::string ProtoFileParser::GetTextFormat( void ProtoFileParser::LogError(const grpc::string& error_msg) { if (!error_msg.empty()) { - std::cout << error_msg << std::endl; + std::cerr << error_msg << std::endl; } has_error_ = true; } diff --git a/test/cpp/util/proto_file_parser.h b/test/cpp/util/proto_file_parser.h index b442d77db98..c8afc4b4b71 100644 --- a/test/cpp/util/proto_file_parser.h +++ b/test/cpp/util/proto_file_parser.h @@ -53,41 +53,57 @@ class ProtoFileParser { // The given proto file_name will be searched in a source tree rooted from // proto_path. The method could be a partial string such as Service.Method or // even just Method. It will log an error if there is ambiguity. - ProtoFileParser(const grpc::string& proto_path, const grpc::string& file_name, - const grpc::string& method); - ProtoFileParser(std::shared_ptr channel, - const grpc::string& method); + const grpc::string& proto_path, + const grpc::string& protofiles); + ~ProtoFileParser(); - grpc::string GetFullMethodName() const { return full_method_name_; } + // Full method name is in the form of Service.Method, it's good to be used in + // descriptor database queries. + grpc::string GetFullMethodName(const grpc::string& method); + + // Formated method name is in the form of /Service/Method, it's good to be + // used as the argument of Stub::Call() + grpc::string GetFormatedMethodName(const grpc::string& method); + + grpc::string GetSerializedProtoFromMethod( + const grpc::string& method, const grpc::string& text_format_proto, + bool is_request); + + grpc::string GetTextFormatFromMethod(const grpc::string& method, + const grpc::string& serialized_proto, + bool is_request); - grpc::string GetSerializedProto(const grpc::string& text_format_proto, - bool is_request); + grpc::string GetSerializedProtoFromMessageType( + const grpc::string& message_type_name, + const grpc::string& text_format_proto); - grpc::string GetTextFormat(const grpc::string& serialized_proto, - bool is_request); + grpc::string GetTextFormatFromMessageType( + const grpc::string& message_type_name, + const grpc::string& serialized_proto); bool HasError() const { return has_error_; } void LogError(const grpc::string& error_msg); private: - void InitProtoFileParser( - const grpc::string& method, - const std::vector services); + grpc::string GetMessageTypeFromMethod(const grpc::string& method, + bool is_request); bool has_error_; grpc::string request_text_; - grpc::string full_method_name_; google::protobuf::compiler::DiskSourceTree source_tree_; std::unique_ptr error_printer_; std::unique_ptr importer_; - std::unique_ptr desc_db_; + std::unique_ptr reflection_db_; + std::unique_ptr file_db_; + std::unique_ptr desc_db_; std::unique_ptr desc_pool_; std::unique_ptr dynamic_factory_; std::unique_ptr request_prototype_; std::unique_ptr response_prototype_; + std::vector service_desc_list_; }; } // namespace testing From 02139a05dc928ac4fa27c7b794a859e42f3b0831 Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 Aug 2016 11:34:21 -0700 Subject: [PATCH 31/43] Add CliCredentials, config_grpc_cli.h --- Makefile | 2 + build.yaml | 4 +- test/cpp/util/cli_credentials.cc | 63 ++++++++++++++ test/cpp/util/cli_credentials.h | 52 ++++++++++++ test/cpp/util/config_grpc_cli.h | 85 +++++++++++++++++++ test/cpp/util/grpc_cli.cc | 13 +-- test/cpp/util/grpc_tool.cc | 85 ++++++++----------- test/cpp/util/grpc_tool.h | 7 +- test/cpp/util/grpc_tool_test.cc | 27 ++++-- test/cpp/util/proto_file_parser.cc | 42 +++++---- test/cpp/util/proto_file_parser.h | 18 ++-- .../proto_reflection_descriptor_database.cc | 2 +- tools/run_tests/sources_and_headers.json | 8 +- .../grpc_cli_libs/grpc_cli_libs.vcxproj | 7 +- .../grpc_cli_libs.vcxproj.filters | 9 ++ 15 files changed, 321 insertions(+), 103 deletions(-) create mode 100644 test/cpp/util/cli_credentials.cc create mode 100644 test/cpp/util/cli_credentials.h create mode 100644 test/cpp/util/config_grpc_cli.h diff --git a/Makefile b/Makefile index c8d11c4f091..afd675fc50f 100644 --- a/Makefile +++ b/Makefile @@ -4299,6 +4299,7 @@ endif LIBGRPC_CLI_LIBS_SRC = \ test/cpp/util/cli_call.cc \ + test/cpp/util/cli_credentials.cc \ test/cpp/util/grpc_tool.cc \ test/cpp/util/proto_file_parser.cc \ test/cpp/util/proto_reflection_descriptor_database.cc \ @@ -15360,6 +15361,7 @@ test/cpp/qps/usage_timer.cc: $(OPENSSL_DEP) test/cpp/util/benchmark_config.cc: $(OPENSSL_DEP) test/cpp/util/byte_buffer_proto_helper.cc: $(OPENSSL_DEP) test/cpp/util/cli_call.cc: $(OPENSSL_DEP) +test/cpp/util/cli_credentials.cc: $(OPENSSL_DEP) test/cpp/util/create_test_channel.cc: $(OPENSSL_DEP) test/cpp/util/grpc_tool.cc: $(OPENSSL_DEP) test/cpp/util/proto_file_parser.cc: $(OPENSSL_DEP) diff --git a/build.yaml b/build.yaml index 104dcf97f67..09a0de2ef0b 100644 --- a/build.yaml +++ b/build.yaml @@ -1052,18 +1052,20 @@ libs: language: c++ headers: - test/cpp/util/cli_call.h + - test/cpp/util/cli_credentials.h + - test/cpp/util/config_grpc_cli.h - test/cpp/util/grpc_tool.h - test/cpp/util/proto_file_parser.h - test/cpp/util/proto_reflection_descriptor_database.h src: - test/cpp/util/cli_call.cc + - test/cpp/util/cli_credentials.cc - test/cpp/util/grpc_tool.cc - test/cpp/util/proto_file_parser.cc - test/cpp/util/proto_reflection_descriptor_database.cc deps: - grpc++_reflection - grpc++ - - grpc_plugin_support - grpc++_test_config - name: grpc_plugin_support build: protoc diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc new file mode 100644 index 00000000000..8de9393e4d4 --- /dev/null +++ b/test/cpp/util/cli_credentials.cc @@ -0,0 +1,63 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "test/cpp/util/cli_credentials.h" + +#include + +DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); +DEFINE_bool(use_auth, false, "Whether to create default google credentials."); + +namespace grpc { +namespace testing { + +std::shared_ptr CliCredentials::GetCredentials() + const { + if (!FLAGS_enable_ssl) { + return grpc::InsecureChannelCredentials(); + } else { + if (FLAGS_use_auth) { + return grpc::GoogleDefaultCredentials(); + } else { + return grpc::SslCredentials(grpc::SslCredentialsOptions()); + } + } +} + +const grpc::string CliCredentials::GetCredentialUsage() const { + return " --enable_ssl ; Set whether to use tls\n" + " --use_auth ; Set whether to create default google" + " credentials\n"; +} +} // namespace testing +} // namespace grpc diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h new file mode 100644 index 00000000000..a351eaaeb1c --- /dev/null +++ b/test/cpp/util/cli_credentials.h @@ -0,0 +1,52 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_TEST_CPP_UTIL_CLI_CREDENTIALS_H +#define GRPC_TEST_CPP_UTIL_CLI_CREDENTIALS_H + +#include +#include + +namespace grpc { +namespace testing { + +class CliCredentials { + public: + virtual std::shared_ptr GetCredentials() const; + virtual const grpc::string GetCredentialUsage() const; +}; + +} // namespace testing +} // namespace grpc + +#endif // GRPC_TEST_CPP_UTIL_CLI_CREDENTIALS_H diff --git a/test/cpp/util/config_grpc_cli.h b/test/cpp/util/config_grpc_cli.h new file mode 100644 index 00000000000..ea8231aa26d --- /dev/null +++ b/test/cpp/util/config_grpc_cli.h @@ -0,0 +1,85 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_TEST_CPP_UTIL_CONFIG_GRPC_CLI_H +#define GRPC_TEST_CPP_UTIL_CONFIG_GRPC_CLI_H + +#include + +#ifndef GRPC_CUSTOM_DYNAMICMESSAGEFACTORY +#include +#define GRPC_CUSTOM_DYNAMICMESSAGEFACTORY \ + ::google::protobuf::DynamicMessageFactory +#endif + +#ifndef GRPC_CUSTOM_DESCRIPTORPOOLDATABASE +#include +#define GRPC_CUSTOM_DESCRIPTORPOOLDATABASE \ + ::google::protobuf::DescriptorPoolDatabase +#define GRPC_CUSTOM_MERGEDDESCRIPTORDATABASE \ + ::google::protobuf::MergedDescriptorDatabase +#endif + +#ifndef GRPC_CUSTOM_TEXTFORMAT +#include +#define GRPC_CUSTOM_TEXTFORMAT ::google::protobuf::TextFormat +#endif + +#ifndef GRPC_CUSTOM_DISKSOURCETREE +#include +#define GRPC_CUSTOM_DISKSOURCETREE ::google::protobuf::compiler::DiskSourceTree +#define GRPC_CUSTOM_IMPORTER ::google::protobuf::compiler::Importer +#define GRPC_CUSTOM_MULTIFILEERRORCOLLECTOR \ + ::google::protobuf::compiler::MultiFileErrorCollector +#endif + +namespace grpc { +namespace protobuf { + +typedef GRPC_CUSTOM_DYNAMICMESSAGEFACTORY DynamicMessageFactory; + +typedef GRPC_CUSTOM_DESCRIPTORPOOLDATABASE DescriptorPoolDatabase; +typedef GRPC_CUSTOM_MERGEDDESCRIPTORDATABASE MergedDescriptorDatabase; + +typedef GRPC_CUSTOM_TEXTFORMAT TextFormat; + +namespace compiler { +typedef GRPC_CUSTOM_DISKSOURCETREE DiskSourceTree; +typedef GRPC_CUSTOM_IMPORTER Importer; +typedef GRPC_CUSTOM_MULTIFILEERRORCOLLECTOR MultiFileErrorCollector; +} // namespace importer + +} // namespace protobuf +} // namespace grpc + +#endif // GRPC_TEST_CPP_UTIL_CONFIG_GRPC_CLI_H diff --git a/test/cpp/util/grpc_cli.cc b/test/cpp/util/grpc_cli.cc index 897ac5ce4a0..fe248601eeb 100644 --- a/test/cpp/util/grpc_cli.cc +++ b/test/cpp/util/grpc_cli.cc @@ -40,7 +40,7 @@ --protofiles=src/proto/grpc/testing/test.proto --enable_ssl=false Options: - 1. --protofiles, use this flag to provide a proto file if the server does + 1. --protofiles, use this flag to provide proto files if the server does does not have the reflection service. 2. --proto_path, if your proto file is not under current working directory, use this flag to provide a search root. It should work similar to the @@ -50,15 +50,17 @@ --metadata="MyHeaderKey1:Value1:MyHeaderKey2:Value2" 4. --enable_ssl, whether to use tls. 5. --use_auth, if set to true, attach a GoogleDefaultCredentials to the call - 6. --input_binary_file, a file containing the serialized request. The file - can be generated by calling something like: + 6. --infile, input filename (defaults to stdin) + 7. --outfile, output filename (defaults to stdout) + 8. --binary_input, use the serialized request as input. The serialized + request can be generated by calling something like: protoc --proto_path=src/proto/grpc/testing/ \ --encode=grpc.testing.SimpleRequest \ src/proto/grpc/testing/messages.proto \ < input.txt > input.bin If this is used and no proto file is provided in the argument list, the method string has to be exact in the form of /package.service/method. - 7. --output_binary_file, a file to write binary format response into, it can + 9. --binary_output, use binary format response as output, it can be later decoded using protoc: protoc --proto_path=src/proto/grpc/testing/ \ --decode=grpc.testing.SimpleResponse \ @@ -72,6 +74,7 @@ #include #include +#include "test/cpp/util/cli_credentials.h" #include "test/cpp/util/grpc_tool.h" #include "test/cpp/util/test_config.h" @@ -93,6 +96,6 @@ int main(int argc, char** argv) { grpc::testing::InitTest(&argc, &argv, true); return grpc::testing::GrpcToolMainLib( - argc, (const char**)argv, + argc, (const char**)argv, grpc::testing::CliCredentials(), std::bind(SimplePrint, FLAGS_outfile, std::placeholders::_1)); } diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index a181ee7597b..17fc406ccab 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -47,14 +47,12 @@ #include #include #include -#include "test/cpp/util/cli_call.h" +#include "test/cpp/util/cli_call.h" #include "test/cpp/util/proto_file_parser.h" #include "test/cpp/util/proto_reflection_descriptor_database.h" #include "test/cpp/util/test_config.h" -DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); DEFINE_bool(remotedb, true, "Use server types to parse and format messages"); DEFINE_string(metadata, "", "Metadata to send to server, in the form of key1:val1:key2:val2"); @@ -73,8 +71,10 @@ class GrpcTool { explicit GrpcTool(); virtual ~GrpcTool() {} - bool Help(int argc, const char** argv, GrpcToolOutputCallback callback); - bool CallMethod(int argc, const char** argv, GrpcToolOutputCallback callback); + bool Help(int argc, const char** argv, CliCredentials cred, + GrpcToolOutputCallback callback); + bool CallMethod(int argc, const char** argv, CliCredentials cred, + GrpcToolOutputCallback callback); // TODO(zyc): implement the following methods // bool ListServices(int argc, const char** argv, GrpcToolOutputCallback // callback); @@ -95,17 +95,18 @@ class GrpcTool { private: void CommandUsage(const grpc::string& usage) const; - std::shared_ptr NewChannel(const grpc::string& server_address); bool print_command_usage_; int usage_exit_status_; + const grpc::string cred_usage_; }; template -std::function -BindWith4Args(T&& func) { +std::function +BindWith5Args(T&& func) { return std::bind(std::forward(func), std::placeholders::_1, std::placeholders::_2, std::placeholders::_3, - std::placeholders::_4); + std::placeholders::_4, std::placeholders::_5); } template @@ -155,21 +156,22 @@ void PrintMetadata(const T& m, const grpc::string& message) { struct Command { const char* command; - std::function + std::function function; int min_args; int max_args; }; const Command ops[] = { - {"help", BindWith4Args(&GrpcTool::Help), 0, INT_MAX}, - // {"ls", BindWith4Args(&GrpcTool::ListServices), 1, 3}, - // {"list", BindWith4Args(&GrpcTool::ListServices), 1, 3}, - {"call", BindWith4Args(&GrpcTool::CallMethod), 2, 3}, - // {"type", BindWith4Args(&GrpcTool::PrintType), 2, 2}, - // {"parse", BindWith4Args(&GrpcTool::ParseMessage), 2, 3}, - // {"totext", BindWith4Args(&GrpcTool::ToText), 2, 3}, - // {"tobinary", BindWith4Args(&GrpcTool::ToBinary), 2, 3}, + {"help", BindWith5Args(&GrpcTool::Help), 0, INT_MAX}, + // {"ls", BindWith5Args(&GrpcTool::ListServices), 1, 3}, + // {"list", BindWith5Args(&GrpcTool::ListServices), 1, 3}, + {"call", BindWith5Args(&GrpcTool::CallMethod), 2, 3}, + // {"type", BindWith5Args(&GrpcTool::PrintType), 2, 2}, + // {"parse", BindWith5Args(&GrpcTool::ParseMessage), 2, 3}, + // {"totext", BindWith5Args(&GrpcTool::ToText), 2, 3}, + // {"tobinary", BindWith5Args(&GrpcTool::ToBinary), 2, 3}, }; void Usage(const grpc::string& msg) { @@ -199,7 +201,7 @@ const Command* FindCommand(const grpc::string& name) { } } // namespace -int GrpcToolMainLib(int argc, const char** argv, +int GrpcToolMainLib(int argc, const char** argv, const CliCredentials cred, GrpcToolOutputCallback callback) { if (argc < 2) { Usage("No command specified"); @@ -216,9 +218,9 @@ int GrpcToolMainLib(int argc, const char** argv, // Force the command to print its usage message fprintf(stderr, "\nWrong number of arguments for %s\n", command.c_str()); grpc_tool.SetPrintCommandMode(1); - return cmd->function(&grpc_tool, -1, NULL, callback); + return cmd->function(&grpc_tool, -1, NULL, cred, callback); } - const bool ok = cmd->function(&grpc_tool, argc, argv, callback); + const bool ok = cmd->function(&grpc_tool, argc, argv, cred, callback); return ok ? 0 : 1; } else { Usage("Invalid command '" + grpc::string(command.c_str()) + "'"); @@ -236,22 +238,7 @@ void GrpcTool::CommandUsage(const grpc::string& usage) const { } } -std::shared_ptr GrpcTool::NewChannel( - const grpc::string& server_address) { - std::shared_ptr creds; - if (!FLAGS_enable_ssl) { - creds = grpc::InsecureChannelCredentials(); - } else { - if (FLAGS_use_auth) { - creds = grpc::GoogleDefaultCredentials(); - } else { - creds = grpc::SslCredentials(grpc::SslCredentialsOptions()); - } - } - return grpc::CreateChannel(server_address, creds); -} - -bool GrpcTool::Help(int argc, const char** argv, +bool GrpcTool::Help(int argc, const char** argv, const CliCredentials cred, GrpcToolOutputCallback callback) { CommandUsage( "Print help\n" @@ -265,12 +252,13 @@ bool GrpcTool::Help(int argc, const char** argv, Usage("Unknown command '" + grpc::string(argv[0]) + "'"); } SetPrintCommandMode(0); - cmd->function(this, -1, NULL, callback); + cmd->function(this, -1, NULL, cred, callback); } return true; } bool GrpcTool::CallMethod(int argc, const char** argv, + const CliCredentials cred, GrpcToolOutputCallback callback) { CommandUsage( "Call method\n" @@ -284,13 +272,11 @@ bool GrpcTool::CallMethod(int argc, const char** argv, " --proto_path ; The search path of proto files, valid" " only when --proto_file is given\n" " --metadata ; The metadata to be sent to the server\n" - " --enable_ssl ; Set whether to use tls\n" - " --use_auth ; Set whether to create default google" - " credentials\n" " --infile ; Input filename (defaults to stdin)\n" " --outfile ; Output filename (defaults to stdout)\n" " --binary_input ; Input in binary format\n" - " --binary_output ; Output in binary format\n"); + " --binary_output ; Output in binary format\n" + + cred.GetCredentialUsage()); std::stringstream output_ss; grpc::string request_text; @@ -319,7 +305,8 @@ bool GrpcTool::CallMethod(int argc, const char** argv, request_text = input_stream.str(); } - std::shared_ptr channel = NewChannel(server_address); + std::shared_ptr channel = + grpc::CreateChannel(server_address, cred.GetCredentials()); if (!FLAGS_binary_input || !FLAGS_binary_output) { parser.reset( new grpc::testing::ProtoFileParser(FLAGS_remotedb ? channel : nullptr, @@ -338,7 +325,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, return false; } } - std::cerr << "connecting to " << server_address << std::endl; + fprintf(stderr, "connecting to %s\n", server_address.c_str()); grpc::string serialized_response_proto; std::multimap client_metadata; @@ -346,7 +333,7 @@ bool GrpcTool::CallMethod(int argc, const char** argv, server_trailing_metadata; ParseMetadataFlag(&client_metadata); PrintMetadata(client_metadata, "Sending client initial metadata:"); - grpc::Status s = grpc::testing::CliCall::Call( + grpc::Status status = grpc::testing::CliCall::Call( channel, parser->GetFormatedMethodName(method_name), serialized_request_proto, &serialized_response_proto, client_metadata, &server_initial_metadata, &server_trailing_metadata); @@ -354,8 +341,8 @@ bool GrpcTool::CallMethod(int argc, const char** argv, "Received initial metadata from server:"); PrintMetadata(server_trailing_metadata, "Received trailing metadata from server:"); - if (s.ok()) { - std::cerr << "Rpc succeeded with OK status" << std::endl; + if (status.ok()) { + fprintf(stderr, "Rpc succeeded with OK status\n"); if (FLAGS_binary_output) { output_ss << serialized_response_proto; } else { @@ -367,8 +354,8 @@ bool GrpcTool::CallMethod(int argc, const char** argv, output_ss << "Response: \n " << response_text << std::endl; } } else { - std::cerr << "Rpc failed with status code " << s.error_code() - << ", error message: " << s.error_message() << std::endl; + fprintf(stderr, "Rpc failed with status code %d, error message: %s\n", + status.error_code(), status.error_message().c_str()); } return callback(output_ss.str()); diff --git a/test/cpp/util/grpc_tool.h b/test/cpp/util/grpc_tool.h index 0c7217d962a..3da86937c2d 100644 --- a/test/cpp/util/grpc_tool.h +++ b/test/cpp/util/grpc_tool.h @@ -34,15 +34,18 @@ #ifndef GRPC_TEST_CPP_UTIL_GRPC_TOOL_H #define GRPC_TEST_CPP_UTIL_GRPC_TOOL_H -#include #include +#include + +#include "test/cpp/util/cli_credentials.h" + namespace grpc { namespace testing { typedef std::function GrpcToolOutputCallback; -int GrpcToolMainLib(int argc, const char **argv, +int GrpcToolMainLib(int argc, const char **argv, CliCredentials cred, GrpcToolOutputCallback callback); } // namespace testing diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 0f3086b4428..b96afaf50cb 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -49,6 +49,7 @@ #include "src/proto/grpc/testing/echo.pb.h" #include "test/core/util/port.h" #include "test/core/util/test_config.h" +#include "test/cpp/util/cli_credentials.h" #include "test/cpp/util/string_ref_helper.h" using grpc::testing::EchoRequest; @@ -56,6 +57,18 @@ using grpc::testing::EchoResponse; namespace grpc { namespace testing { +namespace { + +class TestCliCredentials GRPC_FINAL : public grpc::testing::CliCredentials { + public: + std::shared_ptr GetCredentials() const + GRPC_OVERRIDE { + return InsecureChannelCredentials(); + } + const grpc::string GetCredentialUsage() const GRPC_OVERRIDE { return ""; } +}; + +} // namespame class TestServiceImpl : public ::grpc::testing::EchoTestService::Service { public: @@ -121,7 +134,7 @@ TEST_F(GrpcToolTest, NoCommand) { // Exit with 1, print usage instruction in stderr EXPECT_EXIT( GrpcToolMainLib( - ArraySize(argv), argv, + ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1)), ::testing::ExitedWithCode(1), "No command specified\n" USAGE_REGEX); // No output @@ -135,7 +148,7 @@ TEST_F(GrpcToolTest, InvalidCommand) { // Exit with 1, print usage instruction in stderr EXPECT_EXIT( GrpcToolMainLib( - ArraySize(argv), argv, + ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1)), ::testing::ExitedWithCode(1), "Invalid command 'abc'\n" USAGE_REGEX); // No output @@ -147,7 +160,7 @@ TEST_F(GrpcToolTest, HelpCommand) { std::stringstream output_stream; const char* argv[] = {"grpc_cli", "help"}; // Exit with 1, print usage instruction in stderr - EXPECT_EXIT(GrpcToolMainLib(ArraySize(argv), argv, + EXPECT_EXIT(GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1)), ::testing::ExitedWithCode(1), USAGE_REGEX); @@ -163,7 +176,7 @@ TEST_F(GrpcToolTest, CallCommand) { const char* argv[] = {"grpc_cli", "call", server_address.c_str(), "Echo", "message: 'Hello'"}; - EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, + EXPECT_TRUE(0 == GrpcToolMainLib(ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1))); // Expected output: "message: \"Hello\"" @@ -180,7 +193,7 @@ TEST_F(GrpcToolTest, TooFewArguments) { // Exit with 1 EXPECT_EXIT( GrpcToolMainLib( - ArraySize(argv), argv, + ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1)), ::testing::ExitedWithCode(1), ".*Wrong number of arguments for call.*"); // No output @@ -196,7 +209,7 @@ TEST_F(GrpcToolTest, TooManyArguments) { // Exit with 1 EXPECT_EXIT( GrpcToolMainLib( - ArraySize(argv), argv, + ArraySize(argv), argv, TestCliCredentials(), std::bind(PrintStream, &output_stream, std::placeholders::_1)), ::testing::ExitedWithCode(1), ".*Wrong number of arguments for call.*"); // No output diff --git a/test/cpp/util/proto_file_parser.cc b/test/cpp/util/proto_file_parser.cc index f7ad17e809c..0c88c244486 100644 --- a/test/cpp/util/proto_file_parser.cc +++ b/test/cpp/util/proto_file_parser.cc @@ -37,7 +37,6 @@ #include #include -#include #include namespace grpc { @@ -56,8 +55,7 @@ bool MethodNameMatch(const grpc::string& full_name, const grpc::string& input) { } } // namespace -class ErrorPrinter - : public google::protobuf::compiler::MultiFileErrorCollector { +class ErrorPrinter : public protobuf::compiler::MultiFileErrorCollector { public: explicit ErrorPrinter(ProtoFileParser* parser) : parser_(parser) {} @@ -92,13 +90,12 @@ ProtoFileParser::ProtoFileParser(std::shared_ptr channel, if (!protofiles.empty()) { source_tree_.MapPath("", proto_path); error_printer_.reset(new ErrorPrinter(this)); - importer_.reset(new google::protobuf::compiler::Importer( - &source_tree_, error_printer_.get())); + importer_.reset( + new protobuf::compiler::Importer(&source_tree_, error_printer_.get())); grpc::string file_name; std::stringstream ss(protofiles); while (std::getline(ss, file_name, ',')) { - std::cerr << file_name << std::endl; const auto* file_desc = importer_->Import(file_name); if (file_desc) { for (int i = 0; i < file_desc->service_count(); i++) { @@ -109,8 +106,7 @@ ProtoFileParser::ProtoFileParser(std::shared_ptr channel, } } - file_db_.reset( - new google::protobuf::DescriptorPoolDatabase(*importer_->pool())); + file_db_.reset(new protobuf::DescriptorPoolDatabase(*importer_->pool())); } if (!reflection_db_ && !file_db_) { @@ -123,16 +119,15 @@ ProtoFileParser::ProtoFileParser(std::shared_ptr channel, } else if (!file_db_) { desc_db_ = std::move(reflection_db_); } else { - desc_db_.reset(new google::protobuf::MergedDescriptorDatabase( - reflection_db_.get(), file_db_.get())); + desc_db_.reset(new protobuf::MergedDescriptorDatabase(reflection_db_.get(), + file_db_.get())); } - desc_pool_.reset(new google::protobuf::DescriptorPool(desc_db_.get())); - dynamic_factory_.reset( - new google::protobuf::DynamicMessageFactory(desc_pool_.get())); + desc_pool_.reset(new protobuf::DescriptorPool(desc_db_.get())); + dynamic_factory_.reset(new protobuf::DynamicMessageFactory(desc_pool_.get())); for (auto it = service_list.begin(); it != service_list.end(); it++) { - if (const google::protobuf::ServiceDescriptor* service_desc = + if (const protobuf::ServiceDescriptor* service_desc = desc_pool_->FindServiceByName(*it)) { service_desc_list_.push_back(service_desc); } @@ -143,7 +138,7 @@ ProtoFileParser::~ProtoFileParser() {} grpc::string ProtoFileParser::GetFullMethodName(const grpc::string& method) { has_error_ = false; - const google::protobuf::MethodDescriptor* method_descriptor = nullptr; + const protobuf::MethodDescriptor* method_descriptor = nullptr; for (auto it = service_desc_list_.begin(); it != service_desc_list_.end(); it++) { const auto* service_desc = *it; @@ -192,7 +187,7 @@ grpc::string ProtoFileParser::GetMessageTypeFromMethod( if (has_error_) { return ""; } - const google::protobuf::MethodDescriptor* method_desc = + const protobuf::MethodDescriptor* method_desc = desc_pool_->FindMethodByName(full_method_name); if (!method_desc) { LogError("Method not found"); @@ -231,15 +226,15 @@ grpc::string ProtoFileParser::GetSerializedProtoFromMessageType( const grpc::string& text_format_proto) { has_error_ = false; grpc::string serialized; - const google::protobuf::Descriptor* desc = + const protobuf::Descriptor* desc = desc_pool_->FindMessageTypeByName(message_type_name); if (!desc) { LogError("Message type not found"); return ""; } - grpc::protobuf::Message* msg = dynamic_factory_->GetPrototype(desc)->New(); - bool ok = - google::protobuf::TextFormat::ParseFromString(text_format_proto, msg); + std::unique_ptr msg( + dynamic_factory_->GetPrototype(desc)->New()); + bool ok = protobuf::TextFormat::ParseFromString(text_format_proto, msg.get()); if (!ok) { LogError("Failed to parse text format to proto."); return ""; @@ -256,19 +251,20 @@ grpc::string ProtoFileParser::GetTextFormatFromMessageType( const grpc::string& message_type_name, const grpc::string& serialized_proto) { has_error_ = false; - const google::protobuf::Descriptor* desc = + const protobuf::Descriptor* desc = desc_pool_->FindMessageTypeByName(message_type_name); if (!desc) { LogError("Message type not found"); return ""; } - grpc::protobuf::Message* msg = dynamic_factory_->GetPrototype(desc)->New(); + std::unique_ptr msg( + dynamic_factory_->GetPrototype(desc)->New()); if (!msg->ParseFromString(serialized_proto)) { LogError("Failed to deserialize proto."); return ""; } grpc::string text_format; - if (!google::protobuf::TextFormat::PrintToString(*msg, &text_format)) { + if (!protobuf::TextFormat::PrintToString(*msg.get(), &text_format)) { LogError("Failed to print proto message to text format"); return ""; } diff --git a/test/cpp/util/proto_file_parser.h b/test/cpp/util/proto_file_parser.h index c8afc4b4b71..300b5789bc4 100644 --- a/test/cpp/util/proto_file_parser.h +++ b/test/cpp/util/proto_file_parser.h @@ -36,11 +36,9 @@ #include -#include -#include #include -#include "src/compiler/config.h" +#include "test/cpp/util/config_grpc_cli.h" #include "test/cpp/util/proto_reflection_descriptor_database.h" namespace grpc { @@ -93,17 +91,17 @@ class ProtoFileParser { bool has_error_; grpc::string request_text_; - google::protobuf::compiler::DiskSourceTree source_tree_; + protobuf::compiler::DiskSourceTree source_tree_; std::unique_ptr error_printer_; - std::unique_ptr importer_; + std::unique_ptr importer_; std::unique_ptr reflection_db_; - std::unique_ptr file_db_; - std::unique_ptr desc_db_; - std::unique_ptr desc_pool_; - std::unique_ptr dynamic_factory_; + std::unique_ptr file_db_; + std::unique_ptr desc_db_; + std::unique_ptr desc_pool_; + std::unique_ptr dynamic_factory_; std::unique_ptr request_prototype_; std::unique_ptr response_prototype_; - std::vector service_desc_list_; + std::vector service_desc_list_; }; } // namespace testing diff --git a/test/cpp/util/proto_reflection_descriptor_database.cc b/test/cpp/util/proto_reflection_descriptor_database.cc index 8fd466feb06..f0d14c686a3 100644 --- a/test/cpp/util/proto_reflection_descriptor_database.cc +++ b/test/cpp/util/proto_reflection_descriptor_database.cc @@ -58,7 +58,7 @@ ProtoReflectionDescriptorDatabase::~ProtoReflectionDescriptorDatabase() { stream_->WritesDone(); Status status = stream_->Finish(); if (!status.ok()) { - gpr_log(GPR_ERROR, + gpr_log(GPR_INFO, "ServerReflectionInfo rpc failed. Error code: %d, details: %s", (int)status.error_code(), status.error_message().c_str()); } diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 3c6d9d5433b..8c883c3b152 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4551,11 +4551,12 @@ "deps": [ "grpc++", "grpc++_reflection", - "grpc++_test_config", - "grpc_plugin_support" + "grpc++_test_config" ], "headers": [ "test/cpp/util/cli_call.h", + "test/cpp/util/cli_credentials.h", + "test/cpp/util/config_grpc_cli.h", "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.h", "test/cpp/util/proto_reflection_descriptor_database.h" @@ -4565,6 +4566,9 @@ "src": [ "test/cpp/util/cli_call.cc", "test/cpp/util/cli_call.h", + "test/cpp/util/cli_credentials.cc", + "test/cpp/util/cli_credentials.h", + "test/cpp/util/config_grpc_cli.h", "test/cpp/util/grpc_tool.cc", "test/cpp/util/grpc_tool.h", "test/cpp/util/proto_file_parser.cc", diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj index 09034dc33ef..425b66f155f 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj @@ -148,6 +148,8 @@ + + @@ -155,6 +157,8 @@ + + @@ -169,9 +173,6 @@ {C187A093-A0FE-489D-A40A-6E33DE0F9FEB} - - {B6E81D84-2ACB-41B8-8781-493A944C7817} - {3F7D093D-11F9-C4BC-BEB7-18EB28E3F290} diff --git a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters index cbce2f23120..b2128c282bf 100644 --- a/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_cli_libs/grpc_cli_libs.vcxproj.filters @@ -4,6 +4,9 @@ test\cpp\util + + test\cpp\util + test\cpp\util @@ -18,6 +21,12 @@ test\cpp\util + + test\cpp\util + + + test\cpp\util + test\cpp\util From 3578c5e592e168b64c64d72d9c2af1dba7ec0812 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Mon, 15 Aug 2016 15:13:54 -0700 Subject: [PATCH 32/43] bug fix for fireball app modified condition for trailing metadata. added more information to log message. --- .../cronet/transport/cronet_transport.c | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 8f11ef73792..50f8df52434 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -412,7 +412,7 @@ static void on_read_completed(cronet_bidirectional_stream *stream, char *data, s->state.rs.received_bytes += count; s->state.rs.remaining_bytes -= count; if (s->state.rs.remaining_bytes > 0) { - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs); s->state.state_op_done[OP_READ_REQ_MADE] = true; cronet_bidirectional_stream_read( s->cbs, s->state.rs.read_buffer + s->state.rs.received_bytes, @@ -602,6 +602,9 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, /* we haven't received trailers yet. */ else if (!stream_state->state_callback_received[OP_RECV_TRAILING_METADATA]) result = false; + /* we haven't received on_succeeded yet. */ + else if (!stream_state->state_callback_received[OP_SUCCEEDED]) + result = false; } else if (op_id == OP_SEND_TRAILING_METADATA) { /* already executed */ if (stream_state->state_op_done[OP_SEND_TRAILING_METADATA]) result = false; @@ -699,16 +702,17 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { /* Start new cronet stream. It is destroyed in on_succeeded, on_canceled, * on_failed */ GPR_ASSERT(s->cbs == NULL); - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_create"); s->cbs = cronet_bidirectional_stream_create(s->curr_ct.engine, s->curr_gs, &cronet_callbacks); + CRONET_LOG(GPR_DEBUG, "%p = cronet_bidirectional_stream_create()", s->cbs); char *url; s->header_array.headers = NULL; convert_metadata_to_cronet_headers( stream_op->send_initial_metadata->list.head, s->curr_ct.host, &url, &s->header_array.headers, &s->header_array.count); s->header_array.capacity = s->header_array.count; - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_start %s", url); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_start(%p, %s)", s->cbs, + url); cronet_bidirectional_stream_start(s->cbs, url, 0, "POST", &s->header_array, false); stream_state->state_op_done[OP_SEND_INITIAL_METADATA] = true; @@ -746,8 +750,8 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { int write_buffer_size; create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, &write_buffer_size); - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p)", - stream_state->ws.write_buffer); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, %p)", + s->cbs, stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, write_buffer_size, false); @@ -785,7 +789,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { GPR_ASSERT(stream_state->rs.read_buffer); stream_state->rs.remaining_bytes = stream_state->rs.length_field; stream_state->rs.received_bytes = 0; - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs); stream_state->state_op_done[OP_READ_REQ_MADE] = true; /* Indicates that at least one read request has been made */ cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, @@ -810,7 +814,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { stream_state->rs.read_buffer = stream_state->rs.grpc_header_bytes; stream_state->rs.remaining_bytes = GRPC_HEADER_SIZE_IN_BYTES; stream_state->rs.received_bytes = 0; - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_read(%p)", s->cbs); stream_state->state_op_done[OP_READ_REQ_MADE] = true; /* Indicates that at least one read request has been made */ cronet_bidirectional_stream_read(s->cbs, stream_state->rs.read_buffer, @@ -857,7 +861,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_TRAILING_METADATA)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_SEND_TRAILING_METADATA", oas); - CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (0)"); + CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, 0)", s->cbs); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, "", 0, true); stream_state->state_op_done[OP_SEND_TRAILING_METADATA] = true; From 302763f8989595c62b6c4b7e1ef851d85d414c0d Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 Aug 2016 17:25:57 -0700 Subject: [PATCH 33/43] Address review comments --- test/cpp/util/cli_credentials.h | 1 + test/cpp/util/proto_file_parser.h | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index a351eaaeb1c..299b8ff6cfb 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -42,6 +42,7 @@ namespace testing { class CliCredentials { public: + virtual ~CliCredentials() {}; virtual std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; }; diff --git a/test/cpp/util/proto_file_parser.h b/test/cpp/util/proto_file_parser.h index 300b5789bc4..b9e4a1ec2b9 100644 --- a/test/cpp/util/proto_file_parser.h +++ b/test/cpp/util/proto_file_parser.h @@ -48,15 +48,18 @@ class ErrorPrinter; // Find method and associated request/response types. class ProtoFileParser { public: - // The given proto file_name will be searched in a source tree rooted from - // proto_path. The method could be a partial string such as Service.Method or - // even just Method. It will log an error if there is ambiguity. - ProtoFileParser(std::shared_ptr channel, - const grpc::string& proto_path, - const grpc::string& protofiles); + // The parser will search proto files using the server reflection service + // provided on the given channel. The given protofiles in a source tree rooted + // from proto_path will also be searched. + explicit ProtoFileParser(std::shared_ptr channel, + const grpc::string& proto_path, + const grpc::string& protofiles); ~ProtoFileParser(); + // The input method name in the following four functions could be a partial + // string such as Service.Method or even just Method. It will log an error if + // there is ambiguity. // Full method name is in the form of Service.Method, it's good to be used in // descriptor database queries. grpc::string GetFullMethodName(const grpc::string& method); From 9bf906234e6e1245e54185a84f132e1fe462757e Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 Aug 2016 17:28:13 -0700 Subject: [PATCH 34/43] Clang-format --- test/cpp/util/cli_credentials.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 299b8ff6cfb..003eddee2b7 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -42,7 +42,7 @@ namespace testing { class CliCredentials { public: - virtual ~CliCredentials() {}; + virtual ~CliCredentials(){}; virtual std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; }; From 2bf991332364cf3e293f66d71e99f274e5959a2e Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Mon, 15 Aug 2016 18:34:43 -0700 Subject: [PATCH 35/43] Fix wrongly included header --- test/cpp/util/grpc_tool.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cpp/util/grpc_tool.cc b/test/cpp/util/grpc_tool.cc index 17fc406ccab..f06053ca23e 100644 --- a/test/cpp/util/grpc_tool.cc +++ b/test/cpp/util/grpc_tool.cc @@ -31,7 +31,7 @@ * */ -#include "grpc_tool.h" +#include "test/cpp/util/grpc_tool.h" #include #include From 4998e30bb4c32a7774d4c8c09735dc7b4ef0caf6 Mon Sep 17 00:00:00 2001 From: Sree Kuchibhotla Date: Tue, 16 Aug 2016 07:26:31 -0700 Subject: [PATCH 36/43] Include missing header in ev_epoll_linux.c --- src/core/lib/iomgr/ev_epoll_linux.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/lib/iomgr/ev_epoll_linux.c b/src/core/lib/iomgr/ev_epoll_linux.c index 6a63c4d1d18..02bcbaa10f5 100644 --- a/src/core/lib/iomgr/ev_epoll_linux.c +++ b/src/core/lib/iomgr/ev_epoll_linux.c @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include From 25aa88af9470de87476e1d4e194d13103826693e Mon Sep 17 00:00:00 2001 From: Yuchen Zeng Date: Tue, 16 Aug 2016 10:50:40 -0700 Subject: [PATCH 37/43] Address review comments --- test/cpp/util/cli_credentials.h | 2 +- test/cpp/util/proto_file_parser.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 003eddee2b7..581b77a9c63 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -42,7 +42,7 @@ namespace testing { class CliCredentials { public: - virtual ~CliCredentials(){}; + virtual ~CliCredentials() {} virtual std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; }; diff --git a/test/cpp/util/proto_file_parser.h b/test/cpp/util/proto_file_parser.h index b9e4a1ec2b9..eda3991e727 100644 --- a/test/cpp/util/proto_file_parser.h +++ b/test/cpp/util/proto_file_parser.h @@ -51,9 +51,9 @@ class ProtoFileParser { // The parser will search proto files using the server reflection service // provided on the given channel. The given protofiles in a source tree rooted // from proto_path will also be searched. - explicit ProtoFileParser(std::shared_ptr channel, - const grpc::string& proto_path, - const grpc::string& protofiles); + ProtoFileParser(std::shared_ptr channel, + const grpc::string& proto_path, + const grpc::string& protofiles); ~ProtoFileParser(); From 1b8deaa407e46a32559ad220c58b26f68e14c1dc Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Tue, 16 Aug 2016 16:29:08 -0700 Subject: [PATCH 38/43] addressed review feedback. --- .../cronet/transport/cronet_transport.c | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 50f8df52434..0fa79870760 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -52,12 +52,12 @@ #define GRPC_HEADER_SIZE_IN_BYTES 5 #define CRONET_LOG(...) \ - { \ + do { \ if (grpc_cronet_trace) gpr_log(__VA_ARGS__); \ - } + } while (0) /* TODO (makdharma): Hook up into the wider tracing mechanism */ -int grpc_cronet_trace = 1; +int grpc_cronet_trace = 0; enum OP_RESULT { ACTION_TAKEN_WITH_CALLBACK, @@ -192,8 +192,6 @@ struct stream_obj { cronet_bidirectional_stream *cbs; cronet_bidirectional_stream_header_array header_array; - /* Used for executing callbacks for ops */ - grpc_exec_ctx exec_ctx; /* Stream level state. Some state will be tracked both at stream and stream_op * level */ struct op_state state; @@ -206,7 +204,8 @@ struct stream_obj { }; typedef struct stream_obj stream_obj; -static enum OP_RESULT execute_stream_op(struct op_and_state *oas); +static enum OP_RESULT execute_stream_op(grpc_exec_ctx *exec_ctx, + struct op_and_state *oas); /* Add a new stream op to op storage. @@ -267,11 +266,12 @@ static void remove_from_storage(struct stream_obj *s, or on the application supplied thread via the perform_stream_op function. */ static void execute_from_storage(stream_obj *s) { + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; gpr_mu_lock(&s->mu); for (struct op_and_state *curr = s->storage.head; curr != NULL;) { CRONET_LOG(GPR_DEBUG, "calling op at %p. done = %d", curr, curr->done); GPR_ASSERT(curr->done == 0); - enum OP_RESULT result = execute_stream_op(curr); + enum OP_RESULT result = execute_stream_op(&exec_ctx, curr); CRONET_LOG(GPR_DEBUG, "execute_stream_op[%p] returns %s", curr, op_result_string[result]); /* if this op is done, then remove it and free memory */ @@ -288,7 +288,7 @@ static void execute_from_storage(stream_obj *s) { } } gpr_mu_unlock(&s->mu); - grpc_exec_ctx_finish(&s->exec_ctx); + grpc_exec_ctx_finish(&exec_ctx); } /* @@ -683,7 +683,8 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, return result; } -static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { +static enum OP_RESULT execute_stream_op(grpc_exec_ctx *exec_ctx, + struct op_and_state *oas) { grpc_transport_stream_op *stream_op = &oas->op; struct stream_obj *s = oas->s; struct op_state *stream_state = &s->state; @@ -724,10 +725,10 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { if (!stream_state->state_op_done[OP_CANCEL_ERROR]) { grpc_chttp2_incoming_metadata_buffer_publish( &oas->s->state.rs.initial_metadata, stream_op->recv_initial_metadata); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_NONE, NULL); } else { - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_initial_metadata_ready, + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_initial_metadata_ready, GRPC_ERROR_CANCELLED, NULL); } stream_state->state_op_done[OP_RECV_INITIAL_METADATA] = true; @@ -764,13 +765,13 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { OP_RECV_MESSAGE)) { CRONET_LOG(GPR_DEBUG, "running: %p OP_RECV_MESSAGE", oas); if (stream_state->state_op_done[OP_CANCEL_ERROR]) { - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_CANCELLED, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; } else if (stream_state->rs.read_stream_closed == true) { /* No more data will be received */ CRONET_LOG(GPR_DEBUG, "read stream closed"); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; @@ -803,7 +804,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; @@ -835,7 +836,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { &stream_state->rs.read_slice_buffer, 0); *((grpc_byte_buffer **)stream_op->recv_message) = (grpc_byte_buffer *)&stream_state->rs.sbs; - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->recv_message_ready, + grpc_exec_ctx_sched(exec_ctx, stream_op->recv_message_ready, GRPC_ERROR_NONE, NULL); stream_state->state_op_done[OP_RECV_MESSAGE] = true; oas->state.state_op_done[OP_RECV_MESSAGE] = true; @@ -882,7 +883,7 @@ static enum OP_RESULT execute_stream_op(struct op_and_state *oas) { /* All actions in this stream_op are complete. Call the on_complete callback */ CRONET_LOG(GPR_DEBUG, "running: %p OP_ON_COMPLETE", oas); - grpc_exec_ctx_sched(&s->exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, + grpc_exec_ctx_sched(exec_ctx, stream_op->on_complete, GRPC_ERROR_NONE, NULL); oas->state.state_op_done[OP_ON_COMPLETE] = true; oas->done = true; @@ -923,7 +924,6 @@ static int init_stream(grpc_exec_ctx *exec_ctx, grpc_transport *gt, memset(s->state.state_callback_received, 0, sizeof(s->state.state_callback_received)); gpr_mu_init(&s->mu); - s->exec_ctx = *exec_ctx; return 0; } From 9088d5954a27efb9048e6591bdce457b27bcfdf2 Mon Sep 17 00:00:00 2001 From: Adele Zhou Date: Tue, 16 Aug 2016 17:48:42 -0700 Subject: [PATCH 39/43] fix_interop_server_arg --- tools/run_tests/run_interop_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 78096b216c6..0d402d67e5e 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -662,7 +662,7 @@ argp.add_argument('--prod_servers', 'cloud_to_prod_auth tests against.')) argp.add_argument('-s', '--server', choices=['all'] + sorted(_SERVERS), - action='append', + nargs='+', help='Run cloud_to_cloud servers in a separate docker ' + 'image. Servers can only be started automatically if ' + '--use_docker option is enabled.', From f8f8f5a2ebad29108b53896e06b354e8d60e4705 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 17 Aug 2016 09:55:46 -0700 Subject: [PATCH 40/43] more review feedback addressed --- .../cronet/transport/cronet_transport.c | 130 +++++++++++------- 1 file changed, 83 insertions(+), 47 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index 0fa79870760..ea131dbc043 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -46,6 +46,7 @@ #include "src/core/lib/support/string.h" #include "src/core/lib/surface/channel.h" #include "src/core/lib/transport/metadata_batch.h" +#include "src/core/lib/transport/static_metadata.h" #include "src/core/lib/transport/transport_impl.h" #include "third_party/objective_c/Cronet/cronet_c_for_grpc.h" @@ -59,18 +60,13 @@ /* TODO (makdharma): Hook up into the wider tracing mechanism */ int grpc_cronet_trace = 0; -enum OP_RESULT { +enum e_op_result { ACTION_TAKEN_WITH_CALLBACK, ACTION_TAKEN_NO_CALLBACK, NO_ACTION_POSSIBLE }; -/* Used for printing debug */ -const char *op_result_string[] = {"ACTION_TAKEN_WITH_CALLBACK", - "ACTION_TAKEN_NO_CALLBACK", - "NO_ACTION_POSSIBLE"}; - -enum OP_ID { +enum e_op_id { OP_SEND_INITIAL_METADATA = 0, OP_SEND_MESSAGE, OP_SEND_TRAILING_METADATA, @@ -87,22 +83,7 @@ enum OP_ID { OP_NUM_OPS }; -const char *op_id_string[] = {"OP_SEND_INITIAL_METADATA", - "OP_SEND_MESSAGE", - "OP_SEND_TRAILING_METADATA", - "OP_RECV_MESSAGE", - "OP_RECV_INITIAL_METADATA", - "OP_RECV_TRAILING_METADATA", - "OP_CANCEL_ERROR", - "OP_ON_COMPLETE", - "OP_FAILED", - "OP_SUCCEEDED", - "OP_CANCELED", - "OP_RECV_MESSAGE_AND_ON_COMPLETE", - "OP_READ_REQ_MADE", - "OP_NUM_OPS"}; - -/* Cronet callbacks */ +/* Cronet callbacks. See cronet_c_for_grpc.h for documentation for each. */ static void on_request_headers_sent(cronet_bidirectional_stream *); static void on_response_headers_received( @@ -134,6 +115,8 @@ struct grpc_cronet_transport { }; typedef struct grpc_cronet_transport grpc_cronet_transport; +/* TODO (makdharma): reorder structure for memory efficiency per + http://www.catb.org/esr/structure-packing/#_structure_reordering: */ struct read_state { /* vars to store data coming from server */ char *read_buffer; @@ -204,14 +187,61 @@ struct stream_obj { }; typedef struct stream_obj stream_obj; -static enum OP_RESULT execute_stream_op(grpc_exec_ctx *exec_ctx, - struct op_and_state *oas); +static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, + struct op_and_state *oas); + +/* + Utility function to translate enum into string for printing +*/ +static const char *op_result_string(enum e_op_result i) { + switch (i) { + case ACTION_TAKEN_WITH_CALLBACK: + return "ACTION_TAKEN_WITH_CALLBACK"; + case ACTION_TAKEN_NO_CALLBACK: + return "ACTION_TAKEN_NO_CALLBACK"; + case NO_ACTION_POSSIBLE: + return "NO_ACTION_POSSIBLE"; + } + GPR_UNREACHABLE_CODE(return "UNKNOWN"); +} + +static const char *op_id_string(enum e_op_id i) { + switch (i) { + case OP_SEND_INITIAL_METADATA: + return "OP_SEND_INITIAL_METADATA"; + case OP_SEND_MESSAGE: + return "OP_SEND_MESSAGE"; + case OP_SEND_TRAILING_METADATA: + return "OP_SEND_TRAILING_METADATA"; + case OP_RECV_MESSAGE: + return "OP_RECV_MESSAGE"; + case OP_RECV_INITIAL_METADATA: + return "OP_RECV_INITIAL_METADATA"; + case OP_RECV_TRAILING_METADATA: + return "OP_RECV_TRAILING_METADATA"; + case OP_CANCEL_ERROR: + return "OP_CANCEL_ERROR"; + case OP_ON_COMPLETE: + return "OP_ON_COMPLETE"; + case OP_FAILED: + return "OP_FAILED"; + case OP_SUCCEEDED: + return "OP_SUCCEEDED"; + case OP_CANCELED: + return "OP_CANCELED"; + case OP_RECV_MESSAGE_AND_ON_COMPLETE: + return "OP_RECV_MESSAGE_AND_ON_COMPLETE"; + case OP_READ_REQ_MADE: + return "OP_READ_REQ_MADE"; + case OP_NUM_OPS: + return "OP_NUM_OPS"; + } +} /* Add a new stream op to op storage. */ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { - gpr_mu_lock(&s->mu); struct op_storage *storage = &s->storage; /* add new op at the beginning of the linked list. The memory is freed in remove_from_storage */ @@ -220,6 +250,7 @@ static void add_to_storage(struct stream_obj *s, grpc_transport_stream_op *op) { memset(&new_op->state, 0, sizeof(new_op->state)); new_op->s = s; new_op->done = false; + gpr_mu_lock(&s->mu); new_op->next = storage->head; storage->head = new_op; storage->num_pending_ops++; @@ -271,9 +302,9 @@ static void execute_from_storage(stream_obj *s) { for (struct op_and_state *curr = s->storage.head; curr != NULL;) { CRONET_LOG(GPR_DEBUG, "calling op at %p. done = %d", curr, curr->done); GPR_ASSERT(curr->done == 0); - enum OP_RESULT result = execute_stream_op(&exec_ctx, curr); + enum e_op_result result = execute_stream_op(&exec_ctx, curr); CRONET_LOG(GPR_DEBUG, "execute_stream_op[%p] returns %s", curr, - op_result_string[result]); + op_result_string(result)); /* if this op is done, then remove it and free memory */ if (curr->done) { struct op_and_state *next = curr->next; @@ -372,8 +403,7 @@ static void on_response_headers_received( memset(&s->state.rs.initial_metadata, 0, sizeof(s->state.rs.initial_metadata)); grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.initial_metadata); - unsigned int i = 0; - for (i = 0; i < headers->count; i++) { + for (size_t i = 0; i < headers->count; i++) { grpc_chttp2_incoming_metadata_buffer_add( &s->state.rs.initial_metadata, grpc_mdelem_from_metadata_strings( @@ -439,8 +469,7 @@ static void on_response_trailers_received( sizeof(s->state.rs.trailing_metadata)); s->state.rs.trailing_metadata_valid = false; grpc_chttp2_incoming_metadata_buffer_init(&s->state.rs.trailing_metadata); - unsigned int i = 0; - for (i = 0; i < trailers->count; i++) { + for (size_t i = 0; i < trailers->count; i++) { CRONET_LOG(GPR_DEBUG, "trailer key=%s, value=%s", trailers->headers[i].key, trailers->headers[i].value); grpc_chttp2_incoming_metadata_buffer_add( @@ -460,10 +489,10 @@ static void on_response_trailers_received( */ static void create_grpc_frame(gpr_slice_buffer *write_slice_buffer, char **pp_write_buffer, - int *p_write_buffer_size) { + size_t *p_write_buffer_size) { gpr_slice slice = gpr_slice_buffer_take_first(write_slice_buffer); size_t length = GPR_SLICE_LENGTH(slice); - *p_write_buffer_size = (int)length + GRPC_HEADER_SIZE_IN_BYTES; + *p_write_buffer_size = length + GRPC_HEADER_SIZE_IN_BYTES; /* This is freed in the on_write_completed callback */ char *write_buffer = gpr_malloc(length + GRPC_HEADER_SIZE_IN_BYTES); *pp_write_buffer = write_buffer; @@ -500,7 +529,8 @@ static void convert_metadata_to_cronet_headers( /* Walk the linked list again, this time copying the header fields. s->num_headers can be less than num_headers_available, as some headers - are not used for cronet + are not used for cronet. + TODO (makdharma): Eliminate need to traverse the LL second time for perf. */ curr = head; int num_headers = 0; @@ -509,12 +539,12 @@ static void convert_metadata_to_cronet_headers( curr = curr->next; const char *key = grpc_mdstr_as_c_string(mdelem->key); const char *value = grpc_mdstr_as_c_string(mdelem->value); - if (strcmp(key, ":scheme") == 0 || strcmp(key, ":method") == 0 || - strcmp(key, ":authority") == 0) { + if (mdelem->key == GRPC_MDSTR_METHOD || mdelem->key == GRPC_MDSTR_SCHEME || + mdelem->key == GRPC_MDSTR_AUTHORITY) { /* Cronet populates these fields on its own */ continue; } - if (strcmp(key, ":path") == 0) { + if (mdelem->key == GRPC_MDSTR_PATH) { /* Create URL by appending :path value to the hostname */ gpr_asprintf(pp_url, "https://%s%s", host, value); continue; @@ -546,13 +576,14 @@ static int parse_grpc_header(const uint8_t *data) { */ static bool op_can_be_run(grpc_transport_stream_op *curr_op, struct op_state *stream_state, - struct op_state *op_state, enum OP_ID op_id) { + struct op_state *op_state, enum e_op_id op_id) { bool result = true; /* When call is canceled, every op can be run, except under following conditions */ - if (stream_state->state_op_done[OP_CANCEL_ERROR] || - stream_state->state_callback_received[OP_FAILED]) { + bool is_canceled_of_failed = stream_state->state_op_done[OP_CANCEL_ERROR] || + stream_state->state_callback_received[OP_FAILED]; + if (is_canceled_of_failed) { if (op_id == OP_SEND_INITIAL_METADATA) result = false; if (op_id == OP_SEND_MESSAGE) result = false; if (op_id == OP_SEND_TRAILING_METADATA) result = false; @@ -678,17 +709,20 @@ static bool op_can_be_run(grpc_transport_stream_op *curr_op, !stream_state->state_callback_received[OP_SEND_MESSAGE]) result = false; } - CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string[op_id], + CRONET_LOG(GPR_DEBUG, "op_can_be_run %s : %s", op_id_string(op_id), result ? "YES" : "NO"); return result; } -static enum OP_RESULT execute_stream_op(grpc_exec_ctx *exec_ctx, - struct op_and_state *oas) { +/* + TODO (makdharma): Break down this function in smaller chunks for readability. +*/ +static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, + struct op_and_state *oas) { grpc_transport_stream_op *stream_op = &oas->op; struct stream_obj *s = oas->s; struct op_state *stream_state = &s->state; - enum OP_RESULT result = NO_ACTION_POSSIBLE; + enum e_op_result result = NO_ACTION_POSSIBLE; if (stream_op->send_initial_metadata && op_can_be_run(stream_op, stream_state, &oas->state, OP_SEND_INITIAL_METADATA)) { @@ -743,19 +777,21 @@ static enum OP_RESULT execute_stream_op(grpc_exec_ctx *exec_ctx, grpc_byte_stream_next(NULL, stream_op->send_message, &slice, stream_op->send_message->length, NULL); /* Check that compression flag is OFF. We don't support compression yet. */ + gpr_log(GPR_ERROR, "Compression is not supported"); GPR_ASSERT(stream_op->send_message->flags == 0); gpr_slice_buffer_add(&write_slice_buffer, slice); + gpr_log(GPR_ERROR, "Empty request is not supported"); GPR_ASSERT(write_slice_buffer.count == 1); /* Empty request not handled yet */ if (write_slice_buffer.count > 0) { - int write_buffer_size; + size_t write_buffer_size; create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, &write_buffer_size); CRONET_LOG(GPR_DEBUG, "cronet_bidirectional_stream_write (%p, %p)", s->cbs, stream_state->ws.write_buffer); stream_state->state_callback_received[OP_SEND_MESSAGE] = false; cronet_bidirectional_stream_write(s->cbs, stream_state->ws.write_buffer, - write_buffer_size, false); + (int)write_buffer_size, false); result = ACTION_TAKEN_WITH_CALLBACK; } stream_state->state_op_done[OP_SEND_MESSAGE] = true; From d8004a86fa4b5b329454701af22238861097736f Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 17 Aug 2016 10:01:36 -0700 Subject: [PATCH 41/43] minor tweak --- .../transport/cronet/transport/cronet_transport.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index ea131dbc043..f7726a971e9 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -777,12 +777,16 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, grpc_byte_stream_next(NULL, stream_op->send_message, &slice, stream_op->send_message->length, NULL); /* Check that compression flag is OFF. We don't support compression yet. */ - gpr_log(GPR_ERROR, "Compression is not supported"); - GPR_ASSERT(stream_op->send_message->flags == 0); + if (stream_op->send_message->flags != 0) { + gpr_log(GPR_ERROR, "Compression is not supported"); + GPR_ASSERT(stream_op->send_message->flags == 0); + } gpr_slice_buffer_add(&write_slice_buffer, slice); - gpr_log(GPR_ERROR, "Empty request is not supported"); - GPR_ASSERT(write_slice_buffer.count == - 1); /* Empty request not handled yet */ + if (write_slice_buffer.count != 1) { + /* Empty request not handled yet */ + gpr_log(GPR_ERROR, "Empty request is not supported"); + GPR_ASSERT(write_slice_buffer.count == 1); + } if (write_slice_buffer.count > 0) { size_t write_buffer_size; create_grpc_frame(&write_slice_buffer, &stream_state->ws.write_buffer, From 3e4be9fc56dc81ce79f60df26a00ca3c5549ae64 Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 17 Aug 2016 14:59:26 -0700 Subject: [PATCH 42/43] fixed compile warnings treated as errors --- src/core/ext/transport/cronet/transport/cronet_transport.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index f7726a971e9..bff9a1e1b98 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -236,6 +236,7 @@ static const char *op_id_string(enum e_op_id i) { case OP_NUM_OPS: return "OP_NUM_OPS"; } + return "UNKNOWN"; } /* @@ -533,7 +534,7 @@ static void convert_metadata_to_cronet_headers( TODO (makdharma): Eliminate need to traverse the LL second time for perf. */ curr = head; - int num_headers = 0; + uint32_t num_headers = 0; while (num_headers < num_headers_available) { grpc_mdelem *mdelem = curr->md; curr = curr->next; @@ -557,7 +558,7 @@ static void convert_metadata_to_cronet_headers( break; } } - *p_num_headers = num_headers; + *p_num_headers = (size_t)num_headers; } static int parse_grpc_header(const uint8_t *data) { @@ -826,7 +827,7 @@ static enum e_op_result execute_stream_op(grpc_exec_ctx *exec_ctx, stream_state->rs.length_field); if (stream_state->rs.length_field > 0) { stream_state->rs.read_buffer = - gpr_malloc(stream_state->rs.length_field); + gpr_malloc((size_t)stream_state->rs.length_field); GPR_ASSERT(stream_state->rs.read_buffer); stream_state->rs.remaining_bytes = stream_state->rs.length_field; stream_state->rs.received_bytes = 0; From 7ca910372b3ce75807b39167c61f284e4c7c705c Mon Sep 17 00:00:00 2001 From: Makarand Dharmapurikar Date: Wed, 17 Aug 2016 15:18:38 -0700 Subject: [PATCH 43/43] changed from uint32_t to size_t --- src/core/ext/transport/cronet/transport/cronet_transport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c index bff9a1e1b98..029c15014ed 100644 --- a/src/core/ext/transport/cronet/transport/cronet_transport.c +++ b/src/core/ext/transport/cronet/transport/cronet_transport.c @@ -516,7 +516,7 @@ static void convert_metadata_to_cronet_headers( cronet_bidirectional_stream_header **pp_headers, size_t *p_num_headers) { grpc_linked_mdelem *curr = head; /* Walk the linked list and get number of header fields */ - uint32_t num_headers_available = 0; + size_t num_headers_available = 0; while (curr != NULL) { curr = curr->next; num_headers_available++; @@ -534,7 +534,7 @@ static void convert_metadata_to_cronet_headers( TODO (makdharma): Eliminate need to traverse the LL second time for perf. */ curr = head; - uint32_t num_headers = 0; + size_t num_headers = 0; while (num_headers < num_headers_available) { grpc_mdelem *mdelem = curr->md; curr = curr->next;