diff --git a/src/google/protobuf/compiler/java/java_extension.cc b/src/google/protobuf/compiler/java/java_extension.cc index 27cf416b20..4db7085e01 100644 --- a/src/google/protobuf/compiler/java/java_extension.cc +++ b/src/google/protobuf/compiler/java/java_extension.cc @@ -181,8 +181,9 @@ void ImmutableExtensionGenerator::Generate(io::Printer* printer) { } } -void ImmutableExtensionGenerator::GenerateNonNestedInitializationCode( +int ImmutableExtensionGenerator::GenerateNonNestedInitializationCode( io::Printer* printer) { + int bytecode_estimate = 0; if (descriptor_->extension_scope() == NULL && HasDescriptorMethods(descriptor_->file())) { // Only applies to non-nested, non-lite extensions. @@ -190,15 +191,18 @@ void ImmutableExtensionGenerator::GenerateNonNestedInitializationCode( "$name$.internalInit(descriptor.getExtensions().get($index$));\n", "name", UnderscoresToCamelCase(descriptor_), "index", SimpleItoa(descriptor_->index())); + bytecode_estimate += 21; } + return bytecode_estimate; } -void ImmutableExtensionGenerator::GenerateRegistrationCode( +int ImmutableExtensionGenerator::GenerateRegistrationCode( io::Printer* printer) { printer->Print( "registry.add($scope$.$name$);\n", "scope", scope_, "name", UnderscoresToCamelCase(descriptor_)); + return 7; } } // namespace java diff --git a/src/google/protobuf/compiler/java/java_extension.h b/src/google/protobuf/compiler/java/java_extension.h index f1701fb571..bdd4226379 100644 --- a/src/google/protobuf/compiler/java/java_extension.h +++ b/src/google/protobuf/compiler/java/java_extension.h @@ -67,8 +67,12 @@ class ExtensionGenerator { virtual ~ExtensionGenerator() {} virtual void Generate(io::Printer* printer) = 0; - virtual void GenerateNonNestedInitializationCode(io::Printer* printer) = 0; - virtual void GenerateRegistrationCode(io::Printer* printer) = 0; + + // Returns an estimate of the number of bytes the printed code will compile to + virtual int GenerateNonNestedInitializationCode(io::Printer* printer) = 0; + + // Returns an estimate of the number of bytes the printed code will compile to + virtual int GenerateRegistrationCode(io::Printer* printer) = 0; protected: static void InitTemplateVars(const FieldDescriptor* descriptor, @@ -88,8 +92,8 @@ class ImmutableExtensionGenerator : public ExtensionGenerator { virtual ~ImmutableExtensionGenerator(); virtual void Generate(io::Printer* printer); - virtual void GenerateNonNestedInitializationCode(io::Printer* printer); - virtual void GenerateRegistrationCode(io::Printer* printer); + virtual int GenerateNonNestedInitializationCode(io::Printer* printer); + virtual int GenerateRegistrationCode(io::Printer* printer); protected: const FieldDescriptor* descriptor_; diff --git a/src/google/protobuf/compiler/java/java_file.cc b/src/google/protobuf/compiler/java/java_file.cc index f1e3cf6788..4a1f452988 100644 --- a/src/google/protobuf/compiler/java/java_file.cc +++ b/src/google/protobuf/compiler/java/java_file.cc @@ -35,6 +35,7 @@ #include #include +#include #ifndef _SHARED_PTR_H #include #endif @@ -62,6 +63,19 @@ namespace java { namespace { +struct FieldDescriptorCompare { + bool operator ()(const FieldDescriptor* f1, const FieldDescriptor* f2) { + if(f1 == NULL) { + return false; + } + if(f2 == NULL) { + return true; + } + return f1->full_name() < f2->full_name(); + } +}; + +typedef std::set FieldDescriptorSet; // Recursively searches the given message to collect extensions. // Returns true if all the extensions can be recognized. The extensions will be @@ -69,7 +83,7 @@ namespace { // Returns false when there are unknown fields, in which case the data in the // extensions output parameter is not reliable and should be discarded. bool CollectExtensions(const Message& message, - vector* extensions) { + FieldDescriptorSet* extensions) { const Reflection* reflection = message.GetReflection(); // There are unknown fields that could be extensions, thus this call fails. @@ -79,7 +93,7 @@ bool CollectExtensions(const Message& message, reflection->ListFields(message, &fields); for (int i = 0; i < fields.size(); i++) { - if (fields[i]->is_extension()) extensions->push_back(fields[i]); + if (fields[i]->is_extension()) extensions->insert(fields[i]); if (GetJavaType(fields[i]) == JAVATYPE_MESSAGE) { if (fields[i]->is_repeated()) { @@ -106,7 +120,7 @@ bool CollectExtensions(const Message& message, // in order to handle this case. void CollectExtensions(const FileDescriptorProto& file_proto, const DescriptorPool& alternate_pool, - vector* extensions, + FieldDescriptorSet* extensions, const string& file_data) { if (!CollectExtensions(file_proto, extensions)) { // There are unknown fields in the file_proto, which are probably @@ -139,6 +153,36 @@ void CollectExtensions(const FileDescriptorProto& file_proto, } } +// Our static initialization methods can become very, very large. +// So large that if we aren't careful we end up blowing the JVM's +// 64K bytes of bytecode/method. Fortunately, since these static +// methods are executed only once near the beginning of a program, +// there's usually plenty of stack space available and we can +// extend our methods by simply chaining them to another method +// with a tail call. This inserts the sequence call-next-method, +// end this one, begin-next-method as needed. +void MaybeRestartJavaMethod(io::Printer* printer, + int *bytecode_estimate, + int *method_num, + const char *chain_statement, + const char *method_decl) { + + // The goal here is to stay under 64K bytes of jvm bytecode/method, + // since otherwise we hit a hardcoded limit in the jvm and javac will + // then fail with the error "code too large". This limit lets our + // estimates be off by a factor of two and still we're okay. + static const int bytesPerMethod = 1<<15; // aka 32K + + if ((*bytecode_estimate) > bytesPerMethod) { + ++(*method_num); + printer->Print(chain_statement, "method_num", SimpleItoa(*method_num)); + printer->Outdent(); + printer->Print("}\n"); + printer->Print(method_decl, "method_num", SimpleItoa(*method_num)); + printer->Indent(); + *bytecode_estimate = 0; + } +} } // namespace @@ -270,9 +314,16 @@ void FileGenerator::Generate(io::Printer* printer) { printer->Print( "static {\n"); printer->Indent(); + int bytecode_estimate = 0; + int method_num = 0; for (int i = 0; i < file_->message_type_count(); i++) { - message_generators_[i]->GenerateStaticVariableInitializers(printer); + bytecode_estimate += message_generators_[i]->GenerateStaticVariableInitializers(printer); + MaybeRestartJavaMethod( + printer, + &bytecode_estimate, &method_num, + "_clinit_autosplit_$method_num$();\n", + "private static void _clinit_autosplit_$method_num$() {\n"); } printer->Outdent(); @@ -303,12 +354,24 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable( SharedCodeGenerator shared_code_generator(file_); shared_code_generator.GenerateDescriptors(printer); + int bytecode_estimate = 0; + int method_num = 0; for (int i = 0; i < file_->message_type_count(); i++) { - message_generators_[i]->GenerateStaticVariableInitializers(printer); + bytecode_estimate += message_generators_[i]->GenerateStaticVariableInitializers(printer); + MaybeRestartJavaMethod( + printer, + &bytecode_estimate, &method_num, + "_clinit_autosplit_dinit_$method_num$();\n", + "private static void _clinit_autosplit_dinit_$method_num$() {\n"); } for (int i = 0; i < file_->extension_count(); i++) { - extension_generators_[i]->GenerateNonNestedInitializationCode(printer); + bytecode_estimate += extension_generators_[i]->GenerateNonNestedInitializationCode(printer); + MaybeRestartJavaMethod( + printer, + &bytecode_estimate, &method_num, + "_clinit_autosplit_dinit_$method_num$();\n", + "private static void _clinit_autosplit_dinit_$method_num$() {\n"); } // Proto compiler builds a DescriptorPool, which holds all the descriptors to @@ -330,7 +393,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable( file_->CopyTo(&file_proto); string file_data; file_proto.SerializeToString(&file_data); - vector extensions; + FieldDescriptorSet extensions; CollectExtensions(file_proto, *file_->pool(), &extensions, file_data); if (extensions.size() > 0) { @@ -339,10 +402,17 @@ void FileGenerator::GenerateDescriptorInitializationCodeForImmutable( printer->Print( "com.google.protobuf.ExtensionRegistry registry =\n" " com.google.protobuf.ExtensionRegistry.newInstance();\n"); - for (int i = 0; i < extensions.size(); i++) { + FieldDescriptorSet::iterator it; + for (it = extensions.begin(); it != extensions.end(); it++) { google::protobuf::scoped_ptr generator( - generator_factory_->NewExtensionGenerator(extensions[i])); - generator->GenerateRegistrationCode(printer); + generator_factory_->NewExtensionGenerator(*it)); + bytecode_estimate += generator->GenerateRegistrationCode(printer); + MaybeRestartJavaMethod( + printer, + &bytecode_estimate, &method_num, + "_clinit_autosplit_dinit_$method_num$(registry);\n", + "private static void _clinit_autosplit_dinit_$method_num$(\n" + " com.google.protobuf.ExtensionRegistry registry) {\n"); } printer->Print( "com.google.protobuf.Descriptors.FileDescriptor\n" @@ -394,7 +464,7 @@ void FileGenerator::GenerateDescriptorInitializationCodeForMutable(io::Printer* file_->CopyTo(&file_proto); string file_data; file_proto.SerializeToString(&file_data); - vector extensions; + FieldDescriptorSet extensions; CollectExtensions(file_proto, *file_->pool(), &extensions, file_data); if (extensions.size() > 0) { diff --git a/src/google/protobuf/compiler/java/java_message.cc b/src/google/protobuf/compiler/java/java_message.cc index e982a17bb9..63df10b4c5 100644 --- a/src/google/protobuf/compiler/java/java_message.cc +++ b/src/google/protobuf/compiler/java/java_message.cc @@ -124,7 +124,7 @@ void ImmutableMessageGenerator::GenerateStaticVariables(io::Printer* printer) { // The descriptor for this type. printer->Print(vars, - "$private$static final com.google.protobuf.Descriptors.Descriptor\n" + "$private$static com.google.protobuf.Descriptors.Descriptor\n" " internal_$identifier$_descriptor;\n"); // And the FieldAccessorTable. @@ -139,8 +139,9 @@ void ImmutableMessageGenerator::GenerateStaticVariables(io::Printer* printer) { } } -void ImmutableMessageGenerator::GenerateStaticVariableInitializers( +int ImmutableMessageGenerator::GenerateStaticVariableInitializers( io::Printer* printer) { + int bytecode_estimate = 0; if (HasDescriptorMethods(descriptor_)) { map vars; vars["identifier"] = UniqueFileScopeIdentifier(descriptor_); @@ -156,22 +157,25 @@ void ImmutableMessageGenerator::GenerateStaticVariableInitializers( printer->Print(vars, "internal_$identifier$_descriptor =\n" " getDescriptor().getMessageTypes().get($index$);\n"); + bytecode_estimate += 30; } else { printer->Print(vars, "internal_$identifier$_descriptor =\n" " internal_$parent$_descriptor.getNestedTypes().get($index$);\n"); + bytecode_estimate += 30; } // And the FieldAccessorTable. - GenerateFieldAccessorTableInitializer(printer); + bytecode_estimate += GenerateFieldAccessorTableInitializer(printer); } // Generate static member initializers for all nested types. for (int i = 0; i < descriptor_->nested_type_count(); i++) { // TODO(kenton): Reuse MessageGenerator objects? - ImmutableMessageGenerator(descriptor_->nested_type(i), context_) + bytecode_estimate += ImmutableMessageGenerator(descriptor_->nested_type(i), context_) .GenerateStaticVariableInitializers(printer); } + return bytecode_estimate; } void ImmutableMessageGenerator:: @@ -191,8 +195,9 @@ GenerateFieldAccessorTable(io::Printer* printer) { " internal_$identifier$_fieldAccessorTable;\n"); } -void ImmutableMessageGenerator:: +int ImmutableMessageGenerator:: GenerateFieldAccessorTableInitializer(io::Printer* printer) { + int bytecode_estimate = 10; printer->Print( "internal_$identifier$_fieldAccessorTable = new\n" " com.google.protobuf.GeneratedMessage.FieldAccessorTable(\n" @@ -203,6 +208,7 @@ GenerateFieldAccessorTableInitializer(io::Printer* printer) { for (int i = 0; i < descriptor_->field_count(); i++) { const FieldDescriptor* field = descriptor_->field(i); const FieldGeneratorInfo* info = context_->GetFieldGeneratorInfo(field); + bytecode_estimate += 6; printer->Print( "\"$field_name$\", ", "field_name", info->capitalized_name); @@ -210,11 +216,13 @@ GenerateFieldAccessorTableInitializer(io::Printer* printer) { for (int i = 0; i < descriptor_->oneof_decl_count(); i++) { const OneofDescriptor* oneof = descriptor_->oneof_decl(i); const OneofGeneratorInfo* info = context_->GetOneofGeneratorInfo(oneof); + bytecode_estimate += 6; printer->Print( "\"$oneof_name$\", ", "oneof_name", info->capitalized_name); } printer->Print("});\n"); + return bytecode_estimate; } // =================================================================== diff --git a/src/google/protobuf/compiler/java/java_message.h b/src/google/protobuf/compiler/java/java_message.h index 016fdd5d74..58dd5f9933 100644 --- a/src/google/protobuf/compiler/java/java_message.h +++ b/src/google/protobuf/compiler/java/java_message.h @@ -67,8 +67,8 @@ class MessageGenerator { virtual void GenerateStaticVariables(io::Printer* printer) = 0; // Output code which initializes the static variables generated by - // GenerateStaticVariables(). - virtual void GenerateStaticVariableInitializers(io::Printer* printer) = 0; + // GenerateStaticVariables(). Returns an estimate of bytecode size. + virtual int GenerateStaticVariableInitializers(io::Printer* printer) = 0; // Generate the class itself. virtual void Generate(io::Printer* printer) = 0; @@ -97,7 +97,9 @@ class ImmutableMessageGenerator : public MessageGenerator { virtual void GenerateInterface(io::Printer* printer); virtual void GenerateExtensionRegistrationCode(io::Printer* printer); virtual void GenerateStaticVariables(io::Printer* printer); - virtual void GenerateStaticVariableInitializers(io::Printer* printer); + + // Returns an estimate of the number of bytes the printed code will compile to + virtual int GenerateStaticVariableInitializers(io::Printer* printer); private: enum UseMemoization { @@ -106,7 +108,9 @@ class ImmutableMessageGenerator : public MessageGenerator { }; void GenerateFieldAccessorTable(io::Printer* printer); - void GenerateFieldAccessorTableInitializer(io::Printer* printer); + + // Returns an estimate of the number of bytes the printed code will compile to + int GenerateFieldAccessorTableInitializer(io::Printer* printer); void GenerateMessageSerializationMethods(io::Printer* printer); void GenerateParseFromMethods(io::Printer* printer);