Chiebot-Mirror
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #101 from fizbin/fix_large_proto_java_code

Browse Source 
Fix java compilation issues when processing large .proto files
changes/07/233107/1
Feng Xiao 10 years ago
parent 048f123ed6 0f1393d66e
commit 03e170446b
5 changed files with 116 additions and 26 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 8
      src/google/protobuf/compiler/java/java_extension.cc
  2. 12
      src/google/protobuf/compiler/java/java_extension.h
  3. 92
      src/google/protobuf/compiler/java/java_file.cc
  4. 18
      src/google/protobuf/compiler/java/java_message.cc
  5. 12
      src/google/protobuf/compiler/java/java_message.h

8
src/google/protobuf/compiler/java/java_extension.cc
Unescape View File

@ -181,8 +181,9 @@ void ImmutableExtensionGenerator::Generate(io::Printer* printer) {
} }
} }
void ImmutableExtensionGenerator::GenerateNonNestedInitializationCode( int ImmutableExtensionGenerator::GenerateNonNestedInitializationCode(
io::Printer* printer) { io::Printer* printer) {
int bytecode_estimate = 0;
if (descriptor_->extension_scope() == NULL && if (descriptor_->extension_scope() == NULL &&
HasDescriptorMethods(descriptor_->file())) { HasDescriptorMethods(descriptor_->file())) {
// Only applies to non-nested, non-lite extensions. // Only applies to non-nested, non-lite extensions.
@ -190,15 +191,18 @@ void ImmutableExtensionGenerator::GenerateNonNestedInitializationCode(
"$name$.internalInit(descriptor.getExtensions().get($index$));\n", "$name$.internalInit(descriptor.getExtensions().get($index$));\n",
"name", UnderscoresToCamelCase(descriptor_), "name", UnderscoresToCamelCase(descriptor_),
"index", SimpleItoa(descriptor_->index())); "index", SimpleItoa(descriptor_->index()));
bytecode_estimate += 21;
} }
return bytecode_estimate;
} }
void ImmutableExtensionGenerator::GenerateRegistrationCode( int ImmutableExtensionGenerator::GenerateRegistrationCode(
io::Printer* printer) { io::Printer* printer) {
printer->Print( printer->Print(
"registry.add($scope$.$name$);\n", "registry.add($scope$.$name$);\n",
"scope", scope_, "scope", scope_,
"name", UnderscoresToCamelCase(descriptor_)); "name", UnderscoresToCamelCase(descriptor_));
return 7;
} }
} // namespace java } // namespace java

12
src/google/protobuf/compiler/java/java_extension.h
Unescape View File

@ -67,8 +67,12 @@ class ExtensionGenerator {
virtual ~ExtensionGenerator() {} virtual ~ExtensionGenerator() {}
virtual void Generate(io::Printer* printer) = 0; virtual void Generate(io::Printer* printer) = 0;
virtual void GenerateNonNestedInitializationCode(io::Printer* printer) = 0;
virtual void GenerateRegistrationCode(io::Printer* printer) = 0; // Returns an estimate of the number of bytes the printed code will compile to
virtual int GenerateNonNestedInitializationCode(io::Printer* printer) = 0;
// Returns an estimate of the number of bytes the printed code will compile to
virtual int GenerateRegistrationCode(io::Printer* printer) = 0;
protected: protected:
static void InitTemplateVars(const FieldDescriptor* descriptor, static void InitTemplateVars(const FieldDescriptor* descriptor,
@ -88,8 +92,8 @@ class ImmutableExtensionGenerator : public ExtensionGenerator {
virtual ~ImmutableExtensionGenerator(); virtual ~ImmutableExtensionGenerator();
virtual void Generate(io::Printer* printer); virtual void Generate(io::Printer* printer);
virtual void GenerateNonNestedInitializationCode(io::Printer* printer); virtual int GenerateNonNestedInitializationCode(io::Printer* printer);
virtual void GenerateRegistrationCode(io::Printer* printer); virtual int GenerateRegistrationCode(io::Printer* printer);
protected: protected:
const FieldDescriptor* descriptor_; const FieldDescriptor* descriptor_;

92
src/google/protobuf/compiler/java/java_file.cc
Unescape View File

@ -35,6 +35,7 @@
#include <google/protobuf/compiler/java/java_file.h> #include <google/protobuf/compiler/java/java_file.h>
#include <memory> #include <memory>
#include <set>
#ifndef _SHARED_PTR_H #ifndef _SHARED_PTR_H
#include <google/protobuf/stubs/shared_ptr.h> #include <google/protobuf/stubs/shared_ptr.h>
#endif #endif
@ -62,6 +63,19 @@ namespace java {
namespace { namespace {
struct FieldDescriptorCompare {
bool operator ()(const FieldDescriptor* f1, const FieldDescriptor* f2) {
if(f1 == NULL) {
return false;
}
if(f2 == NULL) {
return true;
}
return f1->full_name() < f2->full_name();
}
};
typedef std::set<const FieldDescriptor*, FieldDescriptorCompare> FieldDescriptorSet;
// Recursively searches the given message to collect extensions. // Recursively searches the given message to collect extensions.
// Returns true if all the extensions can be recognized. The extensions will be // Returns true if all the extensions can be recognized. The extensions will be
@ -69,7 +83,7 @@ namespace {
// Returns false when there are unknown fields, in which case the data in the // Returns false when there are unknown fields, in which case the data in the
// extensions output parameter is not reliable and should be discarded. // extensions output parameter is not reliable and should be discarded.
bool CollectExtensions(const Message& message, bool CollectExtensions(const Message& message,
vector<const FieldDescriptor*>* extensions) { FieldDescriptorSet* extensions) {
const Reflection* reflection = message.GetReflection(); const Reflection* reflection = message.GetReflection();
// There are unknown fields that could be extensions, thus this call fails. // There are unknown fields that could be extensions, thus this call fails.
@ -79,7 +93,7 @@ bool CollectExtensions(const Message& message,
reflection->ListFields(message, &fields); reflection->ListFields(message, &fields);
for (int i = 0; i < fields.size(); i++) { for (int i = 0; i < fields.size(); i++) {
if (fields[i]->is_extension()) extensions->push_back(fields[i]); if (fields[i]->is_extension()) extensions->insert(fields[i]);
if (GetJavaType(fields[i]) == JAVATYPE_MESSAGE) { if (GetJavaType(fields[i]) == JAVATYPE_MESSAGE) {
if (fields[i]->is_repeated()) { if (fields[i]->is_repeated()) {
@ -106,7 +120,7 @@ bool CollectExtensions(const Message& message,
// in order to handle this case. // in order to handle this case.
void CollectExtensions(const FileDescriptorProto& file_proto, void CollectExtensions(const FileDescriptorProto& file_proto,
const DescriptorPool& alternate_pool, const DescriptorPool& alternate_pool,
vector<const FieldDescriptor*>* extensions, FieldDescriptorSet* extensions,
const string& file_data) { const string& file_data) {
if (!CollectExtensions(file_proto, extensions)) { if (!CollectExtensions(file_proto, extensions)) {
// There are unknown fields in the file_proto, which are probably // There are unknown fields in the file_proto, which are probably
@ -139,6 +153,36 @@ void CollectExtensions(const FileDescriptorProto& file_proto,
} }
} }
// Our static initialization methods can become very, very large.
// So large that if we aren't careful we end up blowing the JVM's
// 64K bytes of bytecode/method. Fortunately, since these static
// methods are executed only once near the beginning of a program,
// there's usually plenty of stack space available and we can
// extend our methods by simply chaining them to another method
// with a tail call. This inserts the sequence call-next-method,
// end this one, begin-next-method as needed.
void MaybeRestartJavaMethod(io::Printer* printer,
int *bytecode_estimate,
int *method_num,
const char *chain_statement,
const char *method_decl) {
// The goal here is to stay under 64K bytes of jvm bytecode/method,
// since otherwise we hit a hardcoded limit in the jvm and javac will
// then fail with the error "code too large". This limit lets our
// estimates be off by a factor of two and still we're okay.
static const int bytesPerMethod = 1<<15; // aka 32K
if ((*bytecode_estimate) > bytesPerMethod) {
++(*method_num);
printer->Print(chain_statement, "method_num", SimpleItoa(*method_num));
printer->Outdent();
printer->Print("}\n");
printer->Print(method_decl, "method_num", SimpleItoa(*method_num));
printer->Indent();
*bytecode_estimate = 0;
}
}
} // namespace } // namespace
@ -270,9 +314,16 @@ void FileGenerator::Generate(io::Printer* printer) {
printer->Print( printer->Print(
"static {\n"); "static {\n");
printer->Indent(); printer->Indent();
int bytecode_estimate = 0;
int method_num = 0;
for (int i = 0; i < file_->message_type_count(); i++) { for (int i = 0; i < file_->message_type_count(); i++) {
message_generators_[i]->GenerateStaticVariableInitializers(printer); bytecode_estimate += message_generators_[i]->GenerateStaticVariableInitializers(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_$method_num$();\n",
"private static void _clinit_autosplit_$method_num$() {\n");
} }
printer->Outdent(); printer->Outdent();
@ -303,12 +354,24 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
SharedCodeGenerator shared_code_generator(file_); SharedCodeGenerator shared_code_generator(file_);
shared_code_generator.GenerateDescriptors(printer); shared_code_generator.GenerateDescriptors(printer);
int bytecode_estimate = 0;
int method_num = 0;
for (int i = 0; i < file_->message_type_count(); i++) { for (int i = 0; i < file_->message_type_count(); i++) {
message_generators_[i]->GenerateStaticVariableInitializers(printer); bytecode_estimate += message_generators_[i]->GenerateStaticVariableInitializers(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$();\n",
"private static void _clinit_autosplit_dinit_$method_num$() {\n");
} }
for (int i = 0; i < file_->extension_count(); i++) { for (int i = 0; i < file_->extension_count(); i++) {
extension_generators_[i]->GenerateNonNestedInitializationCode(printer); bytecode_estimate += extension_generators_[i]->GenerateNonNestedInitializationCode(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$();\n",
"private static void _clinit_autosplit_dinit_$method_num$() {\n");
} }
// Proto compiler builds a DescriptorPool, which holds all the descriptors to // Proto compiler builds a DescriptorPool, which holds all the descriptors to
@ -330,7 +393,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
file_->CopyTo(&file_proto); file_->CopyTo(&file_proto);
string file_data; string file_data;
file_proto.SerializeToString(&file_data); file_proto.SerializeToString(&file_data);
vector<const FieldDescriptor*> extensions; FieldDescriptorSet extensions;
CollectExtensions(file_proto, *file_->pool(), &extensions, file_data); CollectExtensions(file_proto, *file_->pool(), &extensions, file_data);
if (extensions.size() > 0) { if (extensions.size() > 0) {
@ -339,10 +402,17 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable(
printer->Print( printer->Print(
"com.google.protobuf.ExtensionRegistry registry =\n" "com.google.protobuf.ExtensionRegistry registry =\n"
" com.google.protobuf.ExtensionRegistry.newInstance();\n"); " com.google.protobuf.ExtensionRegistry.newInstance();\n");
for (int i = 0; i < extensions.size(); i++) { FieldDescriptorSet::iterator it;
for (it = extensions.begin(); it != extensions.end(); it++) {
google::protobuf::scoped_ptr<ExtensionGenerator> generator( google::protobuf::scoped_ptr<ExtensionGenerator> generator(
generator_factory_->NewExtensionGenerator(extensions[i])); generator_factory_->NewExtensionGenerator(*it));
generator->GenerateRegistrationCode(printer); bytecode_estimate += generator->GenerateRegistrationCode(printer);
MaybeRestartJavaMethod(
printer,
&bytecode_estimate, &method_num,
"_clinit_autosplit_dinit_$method_num$(registry);\n",
"private static void _clinit_autosplit_dinit_$method_num$(\n"
" com.google.protobuf.ExtensionRegistry registry) {\n");
} }
printer->Print( printer->Print(
"com.google.protobuf.Descriptors.FileDescriptor\n" "com.google.protobuf.Descriptors.FileDescriptor\n"
@ -394,7 +464,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer*
file_->CopyTo(&file_proto); file_->CopyTo(&file_proto);
string file_data; string file_data;
file_proto.SerializeToString(&file_data); file_proto.SerializeToString(&file_data);
vector<const FieldDescriptor*> extensions; FieldDescriptorSet extensions;
CollectExtensions(file_proto, *file_->pool(), &extensions, file_data); CollectExtensions(file_proto, *file_->pool(), &extensions, file_data);
if (extensions.size() > 0) { if (extensions.size() > 0) {

18
src/google/protobuf/compiler/java/java_message.cc
Unescape View File

@ -124,7 +124,7 @@ void ImmutableMessageGenerator::GenerateStaticVariables(io::Printer* printer) {
// The descriptor for this type. // The descriptor for this type.
printer->Print(vars, printer->Print(vars,
"$private$static final com.google.protobuf.Descriptors.Descriptor\n" "$private$static com.google.protobuf.Descriptors.Descriptor\n"
" internal_$identifier$_descriptor;\n"); " internal_$identifier$_descriptor;\n");
// And the FieldAccessorTable. // And the FieldAccessorTable.
@ -139,8 +139,9 @@ void ImmutableMessageGenerator::GenerateStaticVariables(io::Printer* printer) {
} }
} }
void ImmutableMessageGenerator::GenerateStaticVariableInitializers( int ImmutableMessageGenerator::GenerateStaticVariableInitializers(
io::Printer* printer) { io::Printer* printer) {
int bytecode_estimate = 0;
if (HasDescriptorMethods(descriptor_)) { if (HasDescriptorMethods(descriptor_)) {
map<string, string> vars; map<string, string> vars;
vars["identifier"] = UniqueFileScopeIdentifier(descriptor_); vars["identifier"] = UniqueFileScopeIdentifier(descriptor_);
@ -156,22 +157,25 @@ void ImmutableMessageGenerator::GenerateStaticVariableInitializers(
printer->Print(vars, printer->Print(vars,
"internal_$identifier$_descriptor =\n" "internal_$identifier$_descriptor =\n"
" getDescriptor().getMessageTypes().get($index$);\n"); " getDescriptor().getMessageTypes().get($index$);\n");
bytecode_estimate += 30;
} else { } else {
printer->Print(vars, printer->Print(vars,
"internal_$identifier$_descriptor =\n" "internal_$identifier$_descriptor =\n"
" internal_$parent$_descriptor.getNestedTypes().get($index$);\n"); " internal_$parent$_descriptor.getNestedTypes().get($index$);\n");
bytecode_estimate += 30;
} }
// And the FieldAccessorTable. // And the FieldAccessorTable.
GenerateFieldAccessorTableInitializer(printer); bytecode_estimate += GenerateFieldAccessorTableInitializer(printer);
} }
// Generate static member initializers for all nested types. // Generate static member initializers for all nested types.
for (int i = 0; i < descriptor_->nested_type_count(); i++) { for (int i = 0; i < descriptor_->nested_type_count(); i++) {
// TODO(kenton): Reuse MessageGenerator objects? // TODO(kenton): Reuse MessageGenerator objects?
ImmutableMessageGenerator(descriptor_->nested_type(i), context_) bytecode_estimate += ImmutableMessageGenerator(descriptor_->nested_type(i), context_)
.GenerateStaticVariableInitializers(printer); .GenerateStaticVariableInitializers(printer);
} }
return bytecode_estimate;
} }
void ImmutableMessageGenerator:: void ImmutableMessageGenerator::
@ -191,8 +195,9 @@ GenerateFieldAccessorTable(io::Printer* printer) {
" internal_$identifier$_fieldAccessorTable;\n"); " internal_$identifier$_fieldAccessorTable;\n");
} }
void ImmutableMessageGenerator:: int ImmutableMessageGenerator::
GenerateFieldAccessorTableInitializer(io::Printer* printer) { GenerateFieldAccessorTableInitializer(io::Printer* printer) {
int bytecode_estimate = 10;
printer->Print( printer->Print(
"internal_$identifier$_fieldAccessorTable = new\n" "internal_$identifier$_fieldAccessorTable = new\n"
" com.google.protobuf.GeneratedMessage.FieldAccessorTable(\n" " com.google.protobuf.GeneratedMessage.FieldAccessorTable(\n"
@ -203,6 +208,7 @@ GenerateFieldAccessorTableInitializer(io::Printer* printer) {
for (int i = 0; i < descriptor_->field_count(); i++) { for (int i = 0; i < descriptor_->field_count(); i++) {
const FieldDescriptor* field = descriptor_->field(i); const FieldDescriptor* field = descriptor_->field(i);
const FieldGeneratorInfo* info = context_->GetFieldGeneratorInfo(field); const FieldGeneratorInfo* info = context_->GetFieldGeneratorInfo(field);
bytecode_estimate += 6;
printer->Print( printer->Print(
"\"$field_name$\", ", "\"$field_name$\", ",
"field_name", info->capitalized_name); "field_name", info->capitalized_name);
@ -210,11 +216,13 @@ GenerateFieldAccessorTableInitializer(io::Printer* printer) {
for (int i = 0; i < descriptor_->oneof_decl_count(); i++) { for (int i = 0; i < descriptor_->oneof_decl_count(); i++) {
const OneofDescriptor* oneof = descriptor_->oneof_decl(i); const OneofDescriptor* oneof = descriptor_->oneof_decl(i);
const OneofGeneratorInfo* info = context_->GetOneofGeneratorInfo(oneof); const OneofGeneratorInfo* info = context_->GetOneofGeneratorInfo(oneof);
bytecode_estimate += 6;
printer->Print( printer->Print(
"\"$oneof_name$\", ", "\"$oneof_name$\", ",
"oneof_name", info->capitalized_name); "oneof_name", info->capitalized_name);
} }
printer->Print("});\n"); printer->Print("});\n");
return bytecode_estimate;
} }
// =================================================================== // ===================================================================

12
src/google/protobuf/compiler/java/java_message.h
Unescape View File

@ -67,8 +67,8 @@ class MessageGenerator {
virtual void GenerateStaticVariables(io::Printer* printer) = 0; virtual void GenerateStaticVariables(io::Printer* printer) = 0;
// Output code which initializes the static variables generated by // Output code which initializes the static variables generated by
// GenerateStaticVariables(). // GenerateStaticVariables(). Returns an estimate of bytecode size.
virtual void GenerateStaticVariableInitializers(io::Printer* printer) = 0; virtual int GenerateStaticVariableInitializers(io::Printer* printer) = 0;
// Generate the class itself. // Generate the class itself.
virtual void Generate(io::Printer* printer) = 0; virtual void Generate(io::Printer* printer) = 0;
@ -97,7 +97,9 @@ class ImmutableMessageGenerator : public MessageGenerator {
virtual void GenerateInterface(io::Printer* printer); virtual void GenerateInterface(io::Printer* printer);
virtual void GenerateExtensionRegistrationCode(io::Printer* printer); virtual void GenerateExtensionRegistrationCode(io::Printer* printer);
virtual void GenerateStaticVariables(io::Printer* printer); virtual void GenerateStaticVariables(io::Printer* printer);
virtual void GenerateStaticVariableInitializers(io::Printer* printer);
// Returns an estimate of the number of bytes the printed code will compile to
virtual int GenerateStaticVariableInitializers(io::Printer* printer);
private: private:
enum UseMemoization { enum UseMemoization {
@ -106,7 +108,9 @@ class ImmutableMessageGenerator : public MessageGenerator {
}; };
void GenerateFieldAccessorTable(io::Printer* printer); void GenerateFieldAccessorTable(io::Printer* printer);
void GenerateFieldAccessorTableInitializer(io::Printer* printer);
// Returns an estimate of the number of bytes the printed code will compile to
int GenerateFieldAccessorTableInitializer(io::Printer* printer);
void GenerateMessageSerializationMethods(io::Printer* printer); void GenerateMessageSerializationMethods(io::Printer* printer);
void GenerateParseFromMethods(io::Printer* printer); void GenerateParseFromMethods(io::Printer* printer);

Write Preview
Loading…
Cancel
Save