From 3bb52151dd4ca829ae827517af3a0fd44e9b0bfb Mon Sep 17 00:00:00 2001 From: Masood Malekghassemi Date: Tue, 17 Mar 2015 21:52:52 -0700 Subject: [PATCH] Make Python package spec indirect This is part of a change to ease internal usage of GRPC. --- Makefile | 46 ++++++++++++++++++--- build.json | 21 +++++++--- src/compiler/python_generator.cc | 69 +++++++++++++++++++++++++++----- src/compiler/python_generator.h | 26 +++++++++++- src/compiler/python_plugin.cc | 52 ++---------------------- 5 files changed, 145 insertions(+), 69 deletions(-) diff --git a/Makefile b/Makefile index 8be6c5fd0a4..b50834dc012 100644 --- a/Makefile +++ b/Makefile @@ -3493,6 +3493,44 @@ $(OBJDIR)/$(CONFIG)/src/cpp/util/status.o: $(OBJDIR)/$(CONFIG)/src/cpp/util/time.o: +LIBGRPC_PYTHON_PLUGIN_SUPPORT_SRC = \ + src/compiler/python_generator.cc \ + +PUBLIC_HEADERS_CXX += \ + src/compiler/python_generator.h \ + +LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_SRC)))) + +ifeq ($(NO_PROTOBUF),true) + +# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay. + +$(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a: protobuf_dep_error + + +else + +$(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a: $(ZLIB_DEP) $(PROTOBUF_DEP) $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS) + $(E) "[AR] Creating $@" + $(Q) mkdir -p `dirname $@` + $(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a + $(Q) $(AR) rcs $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS) +ifeq ($(SYSTEM),Darwin) + $(Q) ranlib $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a +endif + + + + +endif + +ifneq ($(NO_DEPS),true) +-include $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS:.o=.dep) +endif + +$(OBJDIR)/$(CONFIG)/src/compiler/python_generator.o: + + LIBPUBSUB_CLIENT_LIB_SRC = \ $(GENDIR)/examples/pubsub/label.pb.cc \ $(GENDIR)/examples/pubsub/empty.pb.cc \ @@ -8048,7 +8086,6 @@ endif GRPC_PYTHON_PLUGIN_SRC = \ - src/compiler/python_generator.cc \ src/compiler/python_plugin.cc \ GRPC_PYTHON_PLUGIN_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_PYTHON_PLUGIN_SRC)))) @@ -8062,15 +8099,14 @@ $(BINDIR)/$(CONFIG)/grpc_python_plugin: protobuf_dep_error else -$(BINDIR)/$(CONFIG)/grpc_python_plugin: $(PROTOBUF_DEP) $(GRPC_PYTHON_PLUGIN_OBJS) +$(BINDIR)/$(CONFIG)/grpc_python_plugin: $(PROTOBUF_DEP) $(GRPC_PYTHON_PLUGIN_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a $(E) "[HOSTLD] Linking $@" $(Q) mkdir -p `dirname $@` - $(Q) $(HOST_LDXX) $(HOST_LDFLAGS) $(GRPC_PYTHON_PLUGIN_OBJS) $(HOST_LDLIBSXX) $(HOST_LDLIBS_PROTOC) $(HOST_LDLIBS) $(HOST_LDLIBS_PROTOC) -o $(BINDIR)/$(CONFIG)/grpc_python_plugin + $(Q) $(HOST_LDXX) $(HOST_LDFLAGS) $(GRPC_PYTHON_PLUGIN_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a $(HOST_LDLIBSXX) $(HOST_LDLIBS_PROTOC) $(HOST_LDLIBS) $(HOST_LDLIBS_PROTOC) -o $(BINDIR)/$(CONFIG)/grpc_python_plugin endif -$(OBJDIR)/$(CONFIG)/src/compiler/python_generator.o: -$(OBJDIR)/$(CONFIG)/src/compiler/python_plugin.o: +$(OBJDIR)/$(CONFIG)/src/compiler/python_plugin.o: $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a deps_grpc_python_plugin: $(GRPC_PYTHON_PLUGIN_OBJS:.o=.dep) diff --git a/build.json b/build.json index 999e140c5a1..08b23ec5931 100644 --- a/build.json +++ b/build.json @@ -491,6 +491,19 @@ ], "secure": "no" }, + { + "name": "grpc_python_plugin_support", + "build": "protoc", + "language": "c++", + "public_headers": [ + "src/compiler/python_generator.h" + ], + "src": [ + "src/compiler/python_generator.cc" + ], + "deps": [], + "secure": "no" + }, { "name": "pubsub_client_lib", "build": "private", @@ -1739,14 +1752,12 @@ "name": "grpc_python_plugin", "build": "protoc", "language": "c++", - "headers": [ - "src/compiler/python_generator.h" - ], "src": [ - "src/compiler/python_generator.cc", "src/compiler/python_plugin.cc" ], - "deps": [], + "deps": [ + "grpc_python_plugin_support" + ], "secure": "no" }, { diff --git a/src/compiler/python_generator.cc b/src/compiler/python_generator.cc index e4f85450f5a..1ec3772f9f7 100644 --- a/src/compiler/python_generator.cc +++ b/src/compiler/python_generator.cc @@ -36,13 +36,18 @@ #include #include #include +#include #include #include +#include +#include #include #include "src/compiler/generator_helpers.h" #include "src/compiler/python_generator.h" +#include #include +#include #include #include #include @@ -53,8 +58,11 @@ using google::protobuf::Descriptor; using google::protobuf::FileDescriptor; using google::protobuf::MethodDescriptor; using google::protobuf::ServiceDescriptor; +using google::protobuf::compiler::GeneratorContext; +using google::protobuf::io::CodedOutputStream; using google::protobuf::io::Printer; using google::protobuf::io::StringOutputStream; +using google::protobuf::io::ZeroCopyOutputStream; using std::initializer_list; using std::make_pair; using std::map; @@ -63,6 +71,41 @@ using std::replace; using std::vector; namespace grpc_python_generator { + +PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config) + : config_(config) {} + +PythonGrpcGenerator::~PythonGrpcGenerator() {} + +bool PythonGrpcGenerator::Generate( + const FileDescriptor* file, const std::string& parameter, + GeneratorContext* context, std::string* error) const { + // Get output file name. + std::string file_name; + static const int proto_suffix_length = strlen(".proto"); + if (file->name().size() > static_cast(proto_suffix_length) && + file->name().find_last_of(".proto") == file->name().size() - 1) { + file_name = file->name().substr( + 0, file->name().size() - proto_suffix_length) + "_pb2.py"; + } else { + *error = "Invalid proto file name. Proto file must end with .proto"; + return false; + } + + std::unique_ptr output( + context->OpenForInsert(file_name, "module_scope")); + CodedOutputStream coded_out(output.get()); + bool success = false; + std::string code = ""; + tie(success, code) = grpc_python_generator::GetServices(file, config_); + if (success) { + coded_out.WriteRaw(code.data(), code.size()); + return true; + } else { + return false; + } +} + namespace { ////////////////////////////////// // BEGIN FORMATTING BOILERPLATE // @@ -70,7 +113,8 @@ namespace { // Converts an initializer list of the form { key0, value0, key1, value1, ... } // into a map of key* to value*. Is merely a readability helper for later code. -map ListToDict(const initializer_list& values) { +map ListToDict( + const initializer_list& values) { assert(values.size() % 2 == 0); map value_map; auto value_iter = values.begin(); @@ -237,8 +281,10 @@ bool PrintServerFactory(const std::string& package_qualified_service_name, { IndentScope raii_create_server_indent(out); map method_description_constructors; - map> input_message_modules_and_classes; - map> output_message_modules_and_classes; + map> + input_message_modules_and_classes; + map> + output_message_modules_and_classes; for (int i = 0; i < service->method_count(); ++i) { const MethodDescriptor* method = service->method(i); const std::string method_description_constructor = @@ -313,8 +359,10 @@ bool PrintStubFactory(const std::string& package_qualified_service_name, { IndentScope raii_create_server_indent(out); map method_description_constructors; - map> input_message_modules_and_classes; - map> output_message_modules_and_classes; + map> + input_message_modules_and_classes; + map> + output_message_modules_and_classes; for (int i = 0; i < service->method_count(); ++i) { const MethodDescriptor* method = service->method(i); const std::string method_description_constructor = @@ -378,22 +426,25 @@ bool PrintStubFactory(const std::string& package_qualified_service_name, return true; } -bool PrintPreamble(const FileDescriptor* file, Printer* out) { +bool PrintPreamble(const FileDescriptor* file, + const GeneratorConfiguration& config, Printer* out) { out->Print("import abc\n"); - out->Print("from grpc.early_adopter import implementations\n"); + out->Print("from $Package$ import implementations\n", + "Package", config.implementations_package_root); out->Print("from grpc.framework.alpha import utilities\n"); return true; } } // namespace -pair GetServices(const FileDescriptor* file) { +pair GetServices(const FileDescriptor* file, + const GeneratorConfiguration& config) { std::string output; { // Scope the output stream so it closes and finalizes output to the string. StringOutputStream output_stream(&output); Printer out(&output_stream, '$'); - if (!PrintPreamble(file, &out)) { + if (!PrintPreamble(file, config, &out)) { return make_pair(false, ""); } auto package = file->package(); diff --git a/src/compiler/python_generator.h b/src/compiler/python_generator.h index df29ca17e3e..67686dbdfe6 100644 --- a/src/compiler/python_generator.h +++ b/src/compiler/python_generator.h @@ -37,6 +37,9 @@ #include #include +#include +#include + namespace google { namespace protobuf { class FileDescriptor; @@ -45,7 +48,28 @@ class FileDescriptor; namespace grpc_python_generator { -std::pair GetServices(const google::protobuf::FileDescriptor* file); +// Data pertaining to configuration of the generator with respect to anything +// that may be used internally at Google. +struct GeneratorConfiguration { + std::string implementations_package_root; +}; + +class PythonGrpcGenerator : public google::protobuf::compiler::CodeGenerator { + public: + PythonGrpcGenerator(const GeneratorConfiguration& config); + ~PythonGrpcGenerator(); + + bool Generate(const google::protobuf::FileDescriptor* file, + const std::string& parameter, + google::protobuf::compiler::GeneratorContext* context, + std::string* error) const; + private: + GeneratorConfiguration config_; +}; + +std::pair GetServices( + const google::protobuf::FileDescriptor* file, + const GeneratorConfiguration& config); } // namespace grpc_python_generator diff --git a/src/compiler/python_plugin.cc b/src/compiler/python_plugin.cc index e7d172f7bf4..4f59ec91642 100644 --- a/src/compiler/python_plugin.cc +++ b/src/compiler/python_plugin.cc @@ -33,60 +33,14 @@ // Generates a Python gRPC service interface out of Protobuf IDL. -#include -#include -#include -#include - #include "src/compiler/python_generator.h" -#include #include -#include -#include -#include -using google::protobuf::FileDescriptor; -using google::protobuf::compiler::CodeGenerator; -using google::protobuf::compiler::GeneratorContext; using google::protobuf::compiler::PluginMain; -using google::protobuf::io::CodedOutputStream; -using google::protobuf::io::ZeroCopyOutputStream; - -class PythonGrpcGenerator : public CodeGenerator { - public: - PythonGrpcGenerator() {} - ~PythonGrpcGenerator() {} - - bool Generate(const FileDescriptor* file, const std::string& parameter, - GeneratorContext* context, std::string* error) const { - // Get output file name. - std::string file_name; - static const int proto_suffix_length = strlen(".proto"); - if (file->name().size() > static_cast(proto_suffix_length) && - file->name().find_last_of(".proto") == file->name().size() - 1) { - file_name = file->name().substr( - 0, file->name().size() - proto_suffix_length) + "_pb2.py"; - } else { - *error = "Invalid proto file name. Proto file must end with .proto"; - return false; - } - - std::unique_ptr output( - context->OpenForInsert(file_name, "module_scope")); - CodedOutputStream coded_out(output.get()); - bool success = false; - std::string code = ""; - tie(success, code) = grpc_python_generator::GetServices(file); - if (success) { - coded_out.WriteRaw(code.data(), code.size()); - return true; - } else { - return false; - } - } -}; int main(int argc, char* argv[]) { - PythonGrpcGenerator generator; + grpc_python_generator::GeneratorConfiguration config; + config.implementations_package_root = "grpc.early_adopter"; + grpc_python_generator::PythonGrpcGenerator generator(config); return PluginMain(argc, argv, &generator); }