Merge pull request #1081 from soltanmm/internal-codegen-support-s2

Make Python package spec indirect

Makefile

@@ -3575,6 +3575,44 @@ $(OBJDIR)/$(CONFIG)/src/cpp/util/status.o:
 $(OBJDIR)/$(CONFIG)/src/cpp/util/time.o: 
 
 
+LIBGRPC_PYTHON_PLUGIN_SUPPORT_SRC = \
+    src/compiler/python_generator.cc \
+
+PUBLIC_HEADERS_CXX += \
+    src/compiler/python_generator.h \
+
+LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_SRC))))
+
+ifeq ($(NO_PROTOBUF),true)
+
+# You can't build a C++ library if you don't have protobuf - a bit overreached, but still okay.
+
+$(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a: protobuf_dep_error
+
+
+else
+
+$(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a: $(ZLIB_DEP) $(PROTOBUF_DEP) $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS)
+	$(E) "[AR]      Creating $@"
+	$(Q) mkdir -p `dirname $@`
+	$(Q) rm -f $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a
+	$(Q) $(AR) rcs $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS)
+ifeq ($(SYSTEM),Darwin)
+	$(Q) ranlib $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a
+endif
+
+
+
+
+endif
+
+ifneq ($(NO_DEPS),true)
+-include $(LIBGRPC_PYTHON_PLUGIN_SUPPORT_OBJS:.o=.dep)
+endif
+
+$(OBJDIR)/$(CONFIG)/src/compiler/python_generator.o: 
+
+
 LIBPUBSUB_CLIENT_LIB_SRC = \
     $(GENDIR)/examples/pubsub/label.pb.cc \
     $(GENDIR)/examples/pubsub/empty.pb.cc \
@@ -8130,7 +8168,6 @@ endif
 
 
 GRPC_PYTHON_PLUGIN_SRC = \
-    src/compiler/python_generator.cc \
     src/compiler/python_plugin.cc \
 
 GRPC_PYTHON_PLUGIN_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GRPC_PYTHON_PLUGIN_SRC))))
@@ -8144,15 +8181,14 @@ $(BINDIR)/$(CONFIG)/grpc_python_plugin: protobuf_dep_error
 
 else
 
-$(BINDIR)/$(CONFIG)/grpc_python_plugin: $(PROTOBUF_DEP) $(GRPC_PYTHON_PLUGIN_OBJS)
+$(BINDIR)/$(CONFIG)/grpc_python_plugin: $(PROTOBUF_DEP) $(GRPC_PYTHON_PLUGIN_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a
 	$(E) "[HOSTLD]  Linking $@"
 	$(Q) mkdir -p `dirname $@`
-	$(Q) $(HOST_LDXX) $(HOST_LDFLAGS) $(GRPC_PYTHON_PLUGIN_OBJS) $(HOST_LDLIBSXX) $(HOST_LDLIBS_PROTOC) $(HOST_LDLIBS) $(HOST_LDLIBS_PROTOC) -o $(BINDIR)/$(CONFIG)/grpc_python_plugin
+	$(Q) $(HOST_LDXX) $(HOST_LDFLAGS) $(GRPC_PYTHON_PLUGIN_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a $(HOST_LDLIBSXX) $(HOST_LDLIBS_PROTOC) $(HOST_LDLIBS) $(HOST_LDLIBS_PROTOC) -o $(BINDIR)/$(CONFIG)/grpc_python_plugin
 
 endif
 
-$(OBJDIR)/$(CONFIG)/src/compiler/python_generator.o: 
+$(OBJDIR)/$(CONFIG)/src/compiler/python_plugin.o:  $(LIBDIR)/$(CONFIG)/libgrpc_python_plugin_support.a
-$(OBJDIR)/$(CONFIG)/src/compiler/python_plugin.o: 
 
 deps_grpc_python_plugin: $(GRPC_PYTHON_PLUGIN_OBJS:.o=.dep)
 

build.json

@@ -491,6 +491,19 @@
       ],
       "secure": "no"
     },
+    {
+      "name": "grpc_python_plugin_support",
+      "build": "protoc",
+      "language": "c++",
+      "public_headers": [
+        "src/compiler/python_generator.h"
+      ],
+      "src": [
+        "src/compiler/python_generator.cc"
+      ],
+      "deps": [],
+      "secure": "no"
+    },
     {
       "name": "pubsub_client_lib",
       "build": "private",
@@ -1739,14 +1752,12 @@
       "name": "grpc_python_plugin",
       "build": "protoc",
       "language": "c++",
-      "headers": [
-        "src/compiler/python_generator.h"
-      ],
       "src": [
-        "src/compiler/python_generator.cc",
         "src/compiler/python_plugin.cc"
       ],
-      "deps": [],
+      "deps": [
+        "grpc_python_plugin_support"
+      ],
       "secure": "no"
     },
     {

src/compiler/python_generator.cc

@@ -36,13 +36,18 @@
 #include <cctype>
 #include <cstring>
 #include <map>
+#include <memory>
 #include <ostream>
 #include <sstream>
+#include <string>
+#include <tuple>
 #include <vector>
 
 #include "src/compiler/generator_helpers.h"
 #include "src/compiler/python_generator.h"
+#include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/printer.h>
+#include <google/protobuf/io/zero_copy_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl_lite.h>
 #include <google/protobuf/descriptor.pb.h>
 #include <google/protobuf/descriptor.h>
@@ -53,8 +58,11 @@ using google::protobuf::Descriptor;
 using google::protobuf::FileDescriptor;
 using google::protobuf::MethodDescriptor;
 using google::protobuf::ServiceDescriptor;
+using google::protobuf::compiler::GeneratorContext;
+using google::protobuf::io::CodedOutputStream;
 using google::protobuf::io::Printer;
 using google::protobuf::io::StringOutputStream;
+using google::protobuf::io::ZeroCopyOutputStream;
 using std::initializer_list;
 using std::make_pair;
 using std::map;
@@ -63,6 +71,41 @@ using std::replace;
 using std::vector;
 
 namespace grpc_python_generator {
+
+PythonGrpcGenerator::PythonGrpcGenerator(const GeneratorConfiguration& config)
+    : config_(config) {}
+
+PythonGrpcGenerator::~PythonGrpcGenerator() {}
+
+bool PythonGrpcGenerator::Generate(
+    const FileDescriptor* file, const std::string& parameter,
+    GeneratorContext* context, std::string* error) const {
+  // Get output file name.
+  std::string file_name;
+  static const int proto_suffix_length = strlen(".proto");
+  if (file->name().size() > static_cast<size_t>(proto_suffix_length) &&
+      file->name().find_last_of(".proto") == file->name().size() - 1) {
+    file_name = file->name().substr(
+        0, file->name().size() - proto_suffix_length) + "_pb2.py";
+  } else {
+    *error = "Invalid proto file name. Proto file must end with .proto";
+    return false;
+  }
+
+  std::unique_ptr<ZeroCopyOutputStream> output(
+      context->OpenForInsert(file_name, "module_scope"));
+  CodedOutputStream coded_out(output.get());
+  bool success = false;
+  std::string code = "";
+  tie(success, code) = grpc_python_generator::GetServices(file, config_);
+  if (success) {
+    coded_out.WriteRaw(code.data(), code.size());
+    return true;
+  } else {
+    return false;
+  }
+}
+
 namespace {
 //////////////////////////////////
 // BEGIN FORMATTING BOILERPLATE //
@@ -70,7 +113,8 @@ namespace {
 
 // Converts an initializer list of the form { key0, value0, key1, value1, ... }
 // into a map of key* to value*. Is merely a readability helper for later code.
-map<std::string, std::string> ListToDict(const initializer_list<std::string>& values) {
+map<std::string, std::string> ListToDict(
+    const initializer_list<std::string>& values) {
   assert(values.size() % 2 == 0);
   map<std::string, std::string> value_map;
   auto value_iter = values.begin();
@@ -237,8 +281,10 @@ bool PrintServerFactory(const std::string& package_qualified_service_name,
   {
     IndentScope raii_create_server_indent(out);
     map<std::string, std::string> method_description_constructors;
-    map<std::string, pair<std::string, std::string>> input_message_modules_and_classes;
+    map<std::string, pair<std::string, std::string>>
-    map<std::string, pair<std::string, std::string>> output_message_modules_and_classes;
+        input_message_modules_and_classes;
+    map<std::string, pair<std::string, std::string>>
+        output_message_modules_and_classes;
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* method = service->method(i);
       const std::string method_description_constructor =
@@ -313,8 +359,10 @@ bool PrintStubFactory(const std::string& package_qualified_service_name,
   {
     IndentScope raii_create_server_indent(out);
     map<std::string, std::string> method_description_constructors;
-    map<std::string, pair<std::string, std::string>> input_message_modules_and_classes;
+    map<std::string, pair<std::string, std::string>>
-    map<std::string, pair<std::string, std::string>> output_message_modules_and_classes;
+        input_message_modules_and_classes;
+    map<std::string, pair<std::string, std::string>>
+        output_message_modules_and_classes;
     for (int i = 0; i < service->method_count(); ++i) {
       const MethodDescriptor* method = service->method(i);
       const std::string method_description_constructor =
@@ -378,22 +426,25 @@ bool PrintStubFactory(const std::string& package_qualified_service_name,
   return true;
 }
 
-bool PrintPreamble(const FileDescriptor* file, Printer* out) {
+bool PrintPreamble(const FileDescriptor* file,
+                   const GeneratorConfiguration& config, Printer* out) {
   out->Print("import abc\n");
-  out->Print("from grpc.early_adopter import implementations\n");
+  out->Print("from $Package$ import implementations\n",
+             "Package", config.implementations_package_root);
   out->Print("from grpc.framework.alpha import utilities\n");
   return true;
 }
 
 }  // namespace
 
-pair<bool, std::string> GetServices(const FileDescriptor* file) {
+pair<bool, std::string> GetServices(const FileDescriptor* file,
+                                    const GeneratorConfiguration& config) {
   std::string output;
   {
     // Scope the output stream so it closes and finalizes output to the string.
     StringOutputStream output_stream(&output);
     Printer out(&output_stream, '$');
-    if (!PrintPreamble(file, &out)) {
+    if (!PrintPreamble(file, config, &out)) {
       return make_pair(false, "");
     }
     auto package = file->package();

src/compiler/python_generator.h

@@ -37,6 +37,9 @@
 #include <string>
 #include <utility>
 
+#include <google/protobuf/compiler/code_generator.h>
+#include <google/protobuf/descriptor.h>
+
 namespace google {
 namespace protobuf {
 class FileDescriptor;
@@ -45,7 +48,28 @@ class FileDescriptor;
 
 namespace grpc_python_generator {
 
-std::pair<bool, std::string> GetServices(const google::protobuf::FileDescriptor* file);
+// Data pertaining to configuration of the generator with respect to anything
+// that may be used internally at Google.
+struct GeneratorConfiguration {
+  std::string implementations_package_root;
+};
+
+class PythonGrpcGenerator : public google::protobuf::compiler::CodeGenerator {
+ public:
+  PythonGrpcGenerator(const GeneratorConfiguration& config);
+  ~PythonGrpcGenerator();
+
+  bool Generate(const google::protobuf::FileDescriptor* file,
+                const std::string& parameter,
+                google::protobuf::compiler::GeneratorContext* context,
+                std::string* error) const;
+ private:
+  GeneratorConfiguration config_;
+};
+
+std::pair<bool, std::string> GetServices(
+    const google::protobuf::FileDescriptor* file,
+    const GeneratorConfiguration& config);
 
 }  // namespace grpc_python_generator
 

src/compiler/python_plugin.cc

@@ -33,60 +33,14 @@
 
 // Generates a Python gRPC service interface out of Protobuf IDL.
 
-#include <cstring>
-#include <memory>
-#include <string>
-#include <tuple>
-
 #include "src/compiler/python_generator.h"
-#include <google/protobuf/compiler/code_generator.h>
 #include <google/protobuf/compiler/plugin.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream.h>
-#include <google/protobuf/descriptor.h>
 
-using google::protobuf::FileDescriptor;
-using google::protobuf::compiler::CodeGenerator;
-using google::protobuf::compiler::GeneratorContext;
 using google::protobuf::compiler::PluginMain;
-using google::protobuf::io::CodedOutputStream;
-using google::protobuf::io::ZeroCopyOutputStream;
-
-class PythonGrpcGenerator : public CodeGenerator {
- public:
-  PythonGrpcGenerator() {}
-  ~PythonGrpcGenerator() {}
-
-  bool Generate(const FileDescriptor* file, const std::string& parameter,
-                GeneratorContext* context, std::string* error) const {
-    // Get output file name.
-    std::string file_name;
-    static const int proto_suffix_length = strlen(".proto");
-    if (file->name().size() > static_cast<size_t>(proto_suffix_length) &&
-        file->name().find_last_of(".proto") == file->name().size() - 1) {
-      file_name = file->name().substr(
-          0, file->name().size() - proto_suffix_length) + "_pb2.py";
-    } else {
-      *error = "Invalid proto file name. Proto file must end with .proto";
-      return false;
-    }
-
-    std::unique_ptr<ZeroCopyOutputStream> output(
-        context->OpenForInsert(file_name, "module_scope"));
-    CodedOutputStream coded_out(output.get());
-    bool success = false;
-    std::string code = "";
-    tie(success, code) = grpc_python_generator::GetServices(file);
-    if (success) {
-      coded_out.WriteRaw(code.data(), code.size());
-      return true;
-    } else {
-      return false;
-    }
-  }
-};
 
 int main(int argc, char* argv[]) {
-  PythonGrpcGenerator generator;
+  grpc_python_generator::GeneratorConfiguration config;
+  config.implementations_package_root = "grpc.early_adopter";
+  grpc_python_generator::PythonGrpcGenerator generator(config);
   return PluginMain(argc, argv, &generator);
 }