From 1156ee7ea73b768bc52f1a42dfcc42120291e268 Mon Sep 17 00:00:00 2001 From: Joshua Humphries Date: Mon, 26 Mar 2018 16:49:31 -0400 Subject: [PATCH] source code info for interpreted options; fix source code info for extension range options (#4342) * when interpreting options, rewrite file descriptor's source code info - so that interpreted option paths have correct location information - so that corresponding uninterpreted option paths are removed also includes a fix to source code locations for extension range options --- src/google/protobuf/compiler/parser.cc | 65 ++- src/google/protobuf/compiler/parser.h | 10 +- src/google/protobuf/descriptor.cc | 230 ++++++++-- src/google/protobuf/descriptor_unittest.cc | 511 ++++++++++++++++++++- 4 files changed, 748 insertions(+), 68 deletions(-) diff --git a/src/google/protobuf/compiler/parser.cc b/src/google/protobuf/compiler/parser.cc index df81e55f92..5c7047a631 100644 --- a/src/google/protobuf/compiler/parser.cc +++ b/src/google/protobuf/compiler/parser.cc @@ -337,31 +337,42 @@ void Parser::AddError(const string& error) { Parser::LocationRecorder::LocationRecorder(Parser* parser) : parser_(parser), + source_code_info_(parser->source_code_info_), location_(parser_->source_code_info_->add_location()) { location_->add_span(parser_->input_->current().line); location_->add_span(parser_->input_->current().column); } Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent) { - Init(parent); + Init(parent, parent.source_code_info_); +} + +Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent, + int path1, + SourceCodeInfo* source_code_info) { + Init(parent, source_code_info); + AddPath(path1); } Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent, int path1) { - Init(parent); + Init(parent, parent.source_code_info_); AddPath(path1); } Parser::LocationRecorder::LocationRecorder(const LocationRecorder& parent, int path1, int path2) { - Init(parent); + Init(parent, parent.source_code_info_); AddPath(path1); AddPath(path2); } -void Parser::LocationRecorder::Init(const LocationRecorder& parent) { +void Parser::LocationRecorder::Init(const LocationRecorder& parent, + SourceCodeInfo* source_code_info) { parser_ = parent.parser_; - location_ = parser_->source_code_info_->add_location(); + source_code_info_ = source_code_info; + + location_ = source_code_info_->add_location(); location_->mutable_path()->CopyFrom(parent.location_->path()); location_->add_span(parser_->input_->current().line); @@ -403,6 +414,10 @@ void Parser::LocationRecorder::RecordLegacyLocation(const Message* descriptor, } } +int Parser::LocationRecorder::CurrentPathSize() const { + return location_->path_size(); +} + void Parser::LocationRecorder::AttachComments( string* leading, string* trailing, std::vector* detached_comments) const { @@ -1496,21 +1511,30 @@ bool Parser::ParseExtensions(DescriptorProto* message, range->set_end(end); } while (TryConsume(",")); - if (LookingAt("[")) { - LocationRecorder location( - extensions_location, - DescriptorProto::ExtensionRange::kOptionsFieldNumber); - DO(Consume("[")); + if (LookingAt("[")) { + int range_number_index = extensions_location.CurrentPathSize(); + SourceCodeInfo info; // Parse extension range options in the first range. ExtensionRangeOptions* options = message->mutable_extension_range(old_range_size)->mutable_options(); - do { - DO(ParseOption(options, location, containing_file, OPTION_ASSIGNMENT)); - } while (TryConsume(",")); - DO(Consume("]")); + { + LocationRecorder index_location( + extensions_location, 0 /* we fill this in w/ actual index below */, + &info); + LocationRecorder location( + index_location, + DescriptorProto::ExtensionRange::kOptionsFieldNumber); + DO(Consume("[")); + + do { + DO(ParseOption(options, location, containing_file, OPTION_ASSIGNMENT)); + } while (TryConsume(",")); + + DO(Consume("]")); + } // Then copy the extension range options to all of the other ranges we've // parsed. @@ -1518,6 +1542,19 @@ bool Parser::ParseExtensions(DescriptorProto* message, message->mutable_extension_range(i)->mutable_options() ->CopyFrom(*options); } + // and copy source locations to the other ranges, too + for (int i = old_range_size; i < message->extension_range_size(); i++) { + for (int j = 0; j < info.location_size(); j++) { + if (info.location(j).path_size() == range_number_index + 1) { + // this location's path is up to the extension range index, but doesn't + // include options; so it's redundant with location above + continue; + } + SourceCodeInfo_Location* dest = source_code_info_->add_location(); + dest->CopyFrom(info.location(j)); + dest->set_path(range_number_index, i); + } + } } DO(ConsumeEndOfDeclaration(";", &extensions_location)); diff --git a/src/google/protobuf/compiler/parser.h b/src/google/protobuf/compiler/parser.h index 01d8a66ba2..5d98e5e18a 100644 --- a/src/google/protobuf/compiler/parser.h +++ b/src/google/protobuf/compiler/parser.h @@ -224,6 +224,10 @@ class LIBPROTOBUF_EXPORT Parser { LocationRecorder(const LocationRecorder& parent, int path1); LocationRecorder(const LocationRecorder& parent, int path1, int path2); + // Creates a recorder that generates locations into given source code info. + LocationRecorder(const LocationRecorder& parent, int path1, + SourceCodeInfo* source_code_info); + ~LocationRecorder(); // Add a path component. See SourceCodeInfo.Location.path in @@ -250,6 +254,9 @@ class LIBPROTOBUF_EXPORT Parser { void RecordLegacyLocation(const Message* descriptor, DescriptorPool::ErrorCollector::ErrorLocation location); + // Returns the number of path components in the recorder's current location. + int CurrentPathSize() const; + // Attaches leading and trailing comments to the location. The two strings // will be swapped into place, so after this is called *leading and // *trailing will be empty. @@ -264,9 +271,10 @@ class LIBPROTOBUF_EXPORT Parser { // SourceCodeInfo.location repeated field. For top-level elements, // parent_index_ is -1. Parser* parser_; + SourceCodeInfo* source_code_info_; SourceCodeInfo::Location* location_; - void Init(const LocationRecorder& parent); + void Init(const LocationRecorder& parent, SourceCodeInfo* source_code_info); }; // ================================================================= diff --git a/src/google/protobuf/descriptor.cc b/src/google/protobuf/descriptor.cc index bd08581d1c..d466dd8b4e 100644 --- a/src/google/protobuf/descriptor.cc +++ b/src/google/protobuf/descriptor.cc @@ -3118,15 +3118,18 @@ namespace { struct OptionsToInterpret { OptionsToInterpret(const string& ns, const string& el, + std::vector& path, const Message* orig_opt, Message* opt) : name_scope(ns), element_name(el), + element_path(path), original_options(orig_opt), options(opt) { } string name_scope; string element_name; + std::vector element_path; const Message* original_options; Message* options; }; @@ -3293,7 +3296,7 @@ class DescriptorBuilder { // descriptor.proto. template void AllocateOptions( const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + DescriptorT* descriptor, int options_field_tag); // Specialization for FileOptions. void AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor); @@ -3303,7 +3306,8 @@ class DescriptorBuilder { const string& name_scope, const string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor); + DescriptorT* descriptor, + std::vector& options_path); // These methods all have the same signature for the sake of the BUILD_ARRAY // macro, below. @@ -3392,13 +3396,21 @@ class DescriptorBuilder { // Otherwise returns true. bool InterpretOptions(OptionsToInterpret* options_to_interpret); + // Updates the given source code info by re-writing uninterpreted option + // locations to refer to the corresponding interpreted option. + void UpdateSourceCodeInfo(SourceCodeInfo* info); + class AggregateOptionFinder; private: // Interprets uninterpreted_option_ on the specified message, which // must be the mutable copy of the original options message to which - // uninterpreted_option_ belongs. - bool InterpretSingleOption(Message* options); + // uninterpreted_option_ belongs. The given src_path is the source + // location path to the uninterpreted option, and options_path is the + // source location path to the options message. The location paths are + // recorded and then used in UpdateSourceCodeInfo. + bool InterpretSingleOption(Message* options, std::vector& src_path, + std::vector& options_path); // Adds the uninterpreted_option to the given options message verbatim. // Used when AllowUnknownDependencies() is in effect and we can't find @@ -3473,6 +3485,16 @@ class DescriptorBuilder { // can use it to find locations recorded by the parser. const UninterpretedOption* uninterpreted_option_; + // This maps the element path of uninterpreted options to the element path + // of the resulting interpreted option. This is used to modify a file's + // source code info to account for option interpretation. + std::map, std::vector> interpreted_paths_; + + // This maps the path to a repeated option field to the known number of + // elements the field contains. This is used to track the compute the + // index portion of the element path when interpreting a single option. + std::map, int> repeated_option_counts_; + // Factory used to create the dynamic messages we need to parse // any aggregate option values we encounter. DynamicMessageFactory dynamic_factory_; @@ -4090,24 +4112,30 @@ void DescriptorBuilder::ValidateSymbolName( // FileDescriptor. template void DescriptorBuilder::AllocateOptions( const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, int options_field_tag) { + std::vector options_path; + descriptor->GetLocationPath(&options_path); + options_path.push_back(options_field_tag); AllocateOptionsImpl(descriptor->full_name(), descriptor->full_name(), - orig_options, descriptor); + orig_options, descriptor, options_path); } // We specialize for FileDescriptor. void DescriptorBuilder::AllocateOptions(const FileOptions& orig_options, FileDescriptor* descriptor) { + std::vector options_path; + options_path.push_back(FileDescriptorProto::kOptionsFieldNumber); // We add the dummy token so that LookupSymbol does the right thing. AllocateOptionsImpl(descriptor->package() + ".dummy", descriptor->name(), - orig_options, descriptor); + orig_options, descriptor, options_path); } template void DescriptorBuilder::AllocateOptionsImpl( const string& name_scope, const string& element_name, const typename DescriptorT::OptionsType& orig_options, - DescriptorT* descriptor) { + DescriptorT* descriptor, + std::vector& options_path) { // We need to use a dummy pointer to work around a bug in older versions of // GCC. Otherwise, the following two lines could be replaced with: // typename DescriptorT::OptionsType* options = @@ -4130,7 +4158,8 @@ template void DescriptorBuilder::AllocateOptionsImpl( // we're still trying to build it. if (options->uninterpreted_option_size() > 0) { options_to_interpret_.push_back( - OptionsToInterpret(name_scope, element_name, &orig_options, options)); + OptionsToInterpret(name_scope, element_name, options_path, + &orig_options, options)); } } @@ -4268,8 +4297,9 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( result->is_placeholder_ = false; result->finished_building_ = false; + SourceCodeInfo *info = NULL; if (proto.has_source_code_info()) { - SourceCodeInfo *info = tables_->AllocateMessage(); + info = tables_->AllocateMessage(); info->CopyFrom(proto.source_code_info()); result->source_code_info_ = info; } else { @@ -4467,6 +4497,10 @@ FileDescriptor* DescriptorBuilder::BuildFileImpl( option_interpreter.InterpretOptions(&(*iter)); } options_to_interpret_.clear(); + + if (info != NULL) { + option_interpreter.UpdateSourceCodeInfo(info); + } } // Validate options. See comments at InternalSetLazilyBuildDependencies about @@ -4538,7 +4572,8 @@ void DescriptorBuilder::BuildMessage(const DescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + DescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), parent, result->name(), @@ -4918,7 +4953,8 @@ void DescriptorBuilder::BuildFieldOrExtension(const FieldDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + FieldDescriptorProto::kOptionsFieldNumber); } @@ -4952,8 +4988,16 @@ void DescriptorBuilder::BuildExtensionRange( if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { + std::vector options_path; + parent->GetLocationPath(&options_path); + options_path.push_back(DescriptorProto::kExtensionRangeFieldNumber); + // find index of this extension range in order to compute path + int index; + for (index = 0; parent->extension_ranges_ + index != result; index++); + options_path.push_back(index); + options_path.push_back(DescriptorProto_ExtensionRange::kOptionsFieldNumber); AllocateOptionsImpl(parent->full_name(), parent->full_name(), - proto.options(), result); + proto.options(), result, options_path); } } @@ -5005,7 +5049,8 @@ void DescriptorBuilder::BuildOneof(const OneofDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + OneofDescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), parent, result->name(), @@ -5122,7 +5167,8 @@ void DescriptorBuilder::BuildEnum(const EnumDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + EnumDescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), parent, result->name(), @@ -5199,7 +5245,8 @@ void DescriptorBuilder::BuildEnumValue(const EnumValueDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + EnumValueDescriptorProto::kOptionsFieldNumber); } // Again, enum values are weird because we makes them appear as siblings @@ -5266,7 +5313,8 @@ void DescriptorBuilder::BuildService(const ServiceDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + ServiceDescriptorProto::kOptionsFieldNumber); } AddSymbol(result->full_name(), NULL, result->name(), @@ -5294,7 +5342,8 @@ void DescriptorBuilder::BuildMethod(const MethodDescriptorProto& proto, if (!proto.has_options()) { result->options_ = NULL; // Will set to default_instance later. } else { - AllocateOptions(proto.options(), result); + AllocateOptions(proto.options(), result, + MethodDescriptorProto::kOptionsFieldNumber); } result->client_streaming_ = proto.client_streaming(); @@ -6266,6 +6315,9 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( << "No field named \"uninterpreted_option\" in the Options proto."; options->GetReflection()->ClearField(options, uninterpreted_options_field); + std::vector src_path = options_to_interpret->element_path; + src_path.push_back(uninterpreted_options_field->number()); + // Find the uninterpreted_option field in the original options. const FieldDescriptor* original_uninterpreted_options_field = original_options->GetDescriptor()-> @@ -6276,14 +6328,17 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( const int num_uninterpreted_options = original_options->GetReflection()-> FieldSize(*original_options, original_uninterpreted_options_field); for (int i = 0; i < num_uninterpreted_options; ++i) { + src_path.push_back(i); uninterpreted_option_ = down_cast( &original_options->GetReflection()->GetRepeatedMessage( *original_options, original_uninterpreted_options_field, i)); - if (!InterpretSingleOption(options)) { + if (!InterpretSingleOption(options, src_path, + options_to_interpret->element_path)) { // Error already added by InterpretSingleOption(). failed = true; break; } + src_path.pop_back(); } // Reset these, so we don't have any dangling pointers. uninterpreted_option_ = NULL; @@ -6316,7 +6371,7 @@ bool DescriptorBuilder::OptionInterpreter::InterpretOptions( } bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( - Message* options) { + Message* options, std::vector& src_path, std::vector& opts_path) { // First do some basic validation. if (uninterpreted_option_->name_size() == 0) { // This should never happen unless the parser has gone seriously awry or @@ -6362,6 +6417,8 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( std::vector intermediate_fields; string debug_msg_name = ""; + std::vector dest_path = opts_path; + for (int i = 0; i < uninterpreted_option_->name_size(); ++i) { const string& name_part = uninterpreted_option_->name(i).name_part(); if (debug_msg_name.size() > 0) { @@ -6424,19 +6481,24 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( "\" is not a field or extension of message \"" + descriptor->name() + "\"."); } - } else if (i < uninterpreted_option_->name_size() - 1) { - if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { - return AddNameError("Option \"" + debug_msg_name + - "\" is an atomic type, not a message."); - } else if (field->is_repeated()) { - return AddNameError("Option field \"" + debug_msg_name + - "\" is a repeated message. Repeated message " - "options must be initialized using an " - "aggregate value."); - } else { - // Drill down into the submessage. - intermediate_fields.push_back(field); - descriptor = field->message_type(); + } else { + // accumulate field numbers to form path to interpreted option + dest_path.push_back(field->number()); + + if (i < uninterpreted_option_->name_size() - 1) { + if (field->cpp_type() != FieldDescriptor::CPPTYPE_MESSAGE) { + return AddNameError("Option \"" + debug_msg_name + + "\" is an atomic type, not a message."); + } else if (field->is_repeated()) { + return AddNameError("Option field \"" + debug_msg_name + + "\" is a repeated message. Repeated message " + "options must be initialized using an " + "aggregate value."); + } else { + // Drill down into the submessage. + intermediate_fields.push_back(field); + descriptor = field->message_type(); + } } } } @@ -6457,7 +6519,6 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( return false; // ExamineIfOptionIsSet() already added the error. } - // First set the value on the UnknownFieldSet corresponding to the // innermost message. std::unique_ptr unknown_fields(new UnknownFieldSet()); @@ -6503,9 +6564,110 @@ bool DescriptorBuilder::OptionInterpreter::InterpretSingleOption( options->GetReflection()->MutableUnknownFields(options)->MergeFrom( *unknown_fields); + // record the element path of the interpreted option + if (field->is_repeated()) { + int index = repeated_option_counts_[dest_path]++; + dest_path.push_back(index); + } + interpreted_paths_[src_path] = dest_path; + return true; } +void DescriptorBuilder::OptionInterpreter::UpdateSourceCodeInfo( + SourceCodeInfo* info) { + + if (interpreted_paths_.empty()) { + // nothing to do! + return; + } + + // We find locations that match keys in interpreted_paths_ and + // 1) replace the path with the corresponding value in interpreted_paths_ + // 2) remove any subsequent sub-locations (sub-location is one whose path + // has the parent path as a prefix) + // + // To avoid quadratic behavior of removing interior rows as we go, + // we keep a copy. But we don't actually copy anything until we've + // found the first match (so if the source code info has no locations + // that need to be changed, there is zero copy overhead). + + RepeatedPtrField* locs = info->mutable_location(); + RepeatedPtrField new_locs; + bool copying = false; + + std::vector pathv; + bool matched = false; + + for (RepeatedPtrField::iterator loc = locs->begin(); + loc != locs->end(); loc++) { + + if (matched) { + // see if this location is in the range to remove + bool loc_matches = true; + if (loc->path_size() < pathv.size()) { + loc_matches = false; + } else { + for (int j = 0; j < pathv.size(); j++) { + if (loc->path(j) != pathv[j]) { + loc_matches = false; + break; + } + } + } + + if (loc_matches) { + // don't copy this row since it is a sub-location that we're removing + continue; + } + + matched = false; + } + + pathv.clear(); + for (int j = 0; j < loc->path_size(); j++) { + pathv.push_back(loc->path(j)); + } + + std::map, std::vector>::iterator entry = + interpreted_paths_.find(pathv); + + if (entry == interpreted_paths_.end()) { + // not a match + if (copying) { + new_locs.Add()->CopyFrom(*loc); + } + continue; + } + + matched = true; + + if (!copying) { + // initialize the copy we are building + copying = true; + new_locs.Reserve(locs->size()); + for (RepeatedPtrField::iterator it = + locs->begin(); it != loc; it++) { + new_locs.Add()->CopyFrom(*it); + } + } + + // add replacement and update its path + SourceCodeInfo_Location* replacement = new_locs.Add(); + replacement->CopyFrom(*loc); + replacement->clear_path(); + for (std::vector::iterator rit = entry->second.begin(); + rit != entry->second.end(); rit++) { + replacement->add_path(*rit); + } + } + + // if we made a changed copy, put it in place + if (copying) { + *locs = new_locs; + } +} + void DescriptorBuilder::OptionInterpreter::AddWithoutInterpreting( const UninterpretedOption& uninterpreted_option, Message* options) { const FieldDescriptor* field = diff --git a/src/google/protobuf/descriptor_unittest.cc b/src/google/protobuf/descriptor_unittest.cc index 5f8d445c83..54da095a6b 100644 --- a/src/google/protobuf/descriptor_unittest.cc +++ b/src/google/protobuf/descriptor_unittest.cc @@ -6918,40 +6918,95 @@ class SingletonSourceTree : public compiler::SourceTree { const char *const kSourceLocationTestInput = "syntax = \"proto2\";\n" + "option java_package = \"com.foo.bar\";\n" + "option (test_file_opt) = \"foobar\";\n" "message A {\n" - " optional int32 a = 1;\n" + " option (test_msg_opt) = \"foobar\";\n" + " optional int32 a = 1 [deprecated = true];\n" " message B {\n" - " required double b = 1;\n" + " required double b = 1 [(test_field_opt) = \"foobar\"];\n" + " }\n" + " oneof c {\n" + " option (test_oneof_opt) = \"foobar\";\n" + " string d = 2;\n" + " string e = 3;\n" + " string f = 4;\n" " }\n" "}\n" "enum Indecision {\n" - " YES = 1;\n" - " NO = 2;\n" + " option (test_enum_opt) = 21;\n" + " option (test_enum_opt) = 42;\n" + " option (test_enum_opt) = 63;\n" + " YES = 1 [(test_enumval_opt).a = 100];\n" + " NO = 2 [(test_enumval_opt) = {a:200}];\n" " MAYBE = 3;\n" "}\n" "service S {\n" + " option (test_svc_opt) = {a:100};\n" + " option (test_svc_opt) = {a:200};\n" + " option (test_svc_opt) = {a:300};\n" " rpc Method(A) returns (A.B);\n" // Put an empty line here to make the source location range match. "\n" + " rpc OtherMethod(A) returns (A) {\n" + " option deprecated = true;\n" + " option (test_method_opt) = \"foobar\";\n" + " }\n" "}\n" "message MessageWithExtensions {\n" - " extensions 1000 to max;\n" + " extensions 1000 to 2000, 2001 to max [(test_ext_opt) = \"foobar\"];\n" "}\n" "extend MessageWithExtensions {\n" - " optional int32 int32_extension = 1001;\n" + " repeated int32 int32_extension = 1001 [packed=true];\n" "}\n" "message C {\n" " extend MessageWithExtensions {\n" " optional C message_extension = 1002;\n" " }\n" - "}\n"; + "}\n" + "import \"google/protobuf/descriptor.proto\";\n" + "extend google.protobuf.FileOptions {\n" + " optional string test_file_opt = 10101;\n" + "}\n" + "extend google.protobuf.MessageOptions {\n" + " optional string test_msg_opt = 10101;\n" + "}\n" + "extend google.protobuf.FieldOptions {\n" + " optional string test_field_opt = 10101;\n" + "}\n" + "extend google.protobuf.EnumOptions {\n" + " repeated int32 test_enum_opt = 10101;\n" + "}\n" + "extend google.protobuf.EnumValueOptions {\n" + " optional A test_enumval_opt = 10101;\n" + "}\n" + "extend google.protobuf.ServiceOptions {\n" + " repeated A test_svc_opt = 10101;\n" + "}\n" + "extend google.protobuf.MethodOptions {\n" + " optional string test_method_opt = 10101;\n" + "}\n" + "extend google.protobuf.OneofOptions {\n" + " optional string test_oneof_opt = 10101;\n" + "}\n" + "extend google.protobuf.ExtensionRangeOptions {\n" + " optional string test_ext_opt = 10101;\n" + "}\n" + ; class SourceLocationTest : public testing::Test { public: SourceLocationTest() : source_tree_("/test/test.proto", kSourceLocationTestInput), - db_(&source_tree_), - pool_(&db_, &collector_) {} + simple_db_(), + source_tree_db_(&source_tree_), + merged_db_(&simple_db_, &source_tree_db_), + pool_(&merged_db_, &collector_) { + // we need descriptor.proto to be accessible by the pool + // since our test file imports it + FileDescriptorProto::descriptor()->file()->CopyTo(&file_proto_); + simple_db_.Add(file_proto_); + } static string PrintSourceLocation(const SourceLocation &loc) { return strings::Substitute("$0:$1-$2:$3", @@ -6962,12 +7017,20 @@ class SourceLocationTest : public testing::Test { } private: + FileDescriptorProto file_proto_; AbortingErrorCollector collector_; SingletonSourceTree source_tree_; - compiler::SourceTreeDescriptorDatabase db_; + SimpleDescriptorDatabase simple_db_; // contains descriptor.proto + compiler::SourceTreeDescriptorDatabase source_tree_db_; // loads test.proto + MergedDescriptorDatabase merged_db_; // combines above two dbs protected: DescriptorPool pool_; + + // tag number of all custom options in above test file + static const int kCustomOptionFieldNumber = 10101; + // tag number of field "a" in message type "A" in above test file + static const int kA_aFieldNumber = 1; }; // TODO(adonovan): implement support for option fields and for @@ -6981,27 +7044,27 @@ TEST_F(SourceLocationTest, GetSourceLocation) { const Descriptor *a_desc = file_desc->FindMessageTypeByName("A"); EXPECT_TRUE(a_desc->GetSourceLocation(&loc)); - EXPECT_EQ("2:1-7:2", PrintSourceLocation(loc)); + EXPECT_EQ("4:1-16:2", PrintSourceLocation(loc)); const Descriptor *a_b_desc = a_desc->FindNestedTypeByName("B"); EXPECT_TRUE(a_b_desc->GetSourceLocation(&loc)); - EXPECT_EQ("4:3-6:4", PrintSourceLocation(loc)); + EXPECT_EQ("7:3-9:4", PrintSourceLocation(loc)); const EnumDescriptor *e_desc = file_desc->FindEnumTypeByName("Indecision"); EXPECT_TRUE(e_desc->GetSourceLocation(&loc)); - EXPECT_EQ("8:1-12:2", PrintSourceLocation(loc)); + EXPECT_EQ("17:1-24:2", PrintSourceLocation(loc)); const EnumValueDescriptor *yes_desc = e_desc->FindValueByName("YES"); EXPECT_TRUE(yes_desc->GetSourceLocation(&loc)); - EXPECT_EQ("9:3-9:13", PrintSourceLocation(loc)); + EXPECT_EQ("21:3-21:42", PrintSourceLocation(loc)); const ServiceDescriptor *s_desc = file_desc->FindServiceByName("S"); EXPECT_TRUE(s_desc->GetSourceLocation(&loc)); - EXPECT_EQ("13:1-16:2", PrintSourceLocation(loc)); + EXPECT_EQ("25:1-35:2", PrintSourceLocation(loc)); const MethodDescriptor *m_desc = s_desc->FindMethodByName("Method"); EXPECT_TRUE(m_desc->GetSourceLocation(&loc)); - EXPECT_EQ("14:3-14:31", PrintSourceLocation(loc)); + EXPECT_EQ("29:3-29:31", PrintSourceLocation(loc)); } @@ -7014,16 +7077,426 @@ TEST_F(SourceLocationTest, ExtensionSourceLocation) { const FieldDescriptor *int32_extension_desc = file_desc->FindExtensionByName("int32_extension"); EXPECT_TRUE(int32_extension_desc->GetSourceLocation(&loc)); - EXPECT_EQ("21:3-21:41", PrintSourceLocation(loc)); + EXPECT_EQ("40:3-40:55", PrintSourceLocation(loc)); const Descriptor *c_desc = file_desc->FindMessageTypeByName("C"); EXPECT_TRUE(c_desc->GetSourceLocation(&loc)); - EXPECT_EQ("23:1-27:2", PrintSourceLocation(loc)); + EXPECT_EQ("42:1-46:2", PrintSourceLocation(loc)); const FieldDescriptor *message_extension_desc = c_desc->FindExtensionByName("message_extension"); EXPECT_TRUE(message_extension_desc->GetSourceLocation(&loc)); - EXPECT_EQ("25:5-25:41", PrintSourceLocation(loc)); + EXPECT_EQ("44:5-44:41", PrintSourceLocation(loc)); +} + +TEST_F(SourceLocationTest, InterpretedOptionSourceLocation) { + // This one's a doozy. It checks every kind of option, including + // extension range options. + + // We are verifying that the file's source info contains correct + // info for interpreted options and that it does *not* contain + // any info for corresponding uninterpreted option path. + + SourceLocation loc; + + const FileDescriptor *file_desc = + GOOGLE_CHECK_NOTNULL(pool_.FindFileByName("/test/test.proto")); + + // File options + { + int path[] = {FileDescriptorProto::kOptionsFieldNumber, + FileOptions::kJavaPackageFieldNumber}; + int unint[] = {FileDescriptorProto::kOptionsFieldNumber, + FileOptions::kUninterpretedOptionFieldNumber, + 0}; + + std::vector vpath(path, path + 2); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("2:1-2:37", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 3); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kOptionsFieldNumber, + FileOptions::kUninterpretedOptionFieldNumber, + 1}; + std::vector vpath(path, path + 2); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("3:1-3:35", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 3); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Message option + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kOptionsFieldNumber, + MessageOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 4); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("5:3-5:36", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Field option + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kFieldFieldNumber, + 0, + FieldDescriptorProto::kOptionsFieldNumber, + FieldOptions::kDeprecatedFieldNumber}; + int unint[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kFieldFieldNumber, + 0, + FieldDescriptorProto::kOptionsFieldNumber, + FieldOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("6:25-6:42", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Nested message option + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kNestedTypeFieldNumber, + 0, + DescriptorProto::kFieldFieldNumber, + 0, + FieldDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kNestedTypeFieldNumber, + 0, + DescriptorProto::kFieldFieldNumber, + 0, + FieldDescriptorProto::kOptionsFieldNumber, + FieldOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 8); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("8:28-8:55", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 9); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // One-of option + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kOneofDeclFieldNumber, + 0, + OneofDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 0, + DescriptorProto::kOneofDeclFieldNumber, + 0, + OneofDescriptorProto::kOptionsFieldNumber, + OneofOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("11:5-11:40", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Enum option, repeated options + { + int path[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + 0}; + int unint[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kOptionsFieldNumber, + EnumOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("18:3-18:31", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + 1}; + int unint[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kOptionsFieldNumber, + EnumOptions::kUninterpretedOptionFieldNumber, + 1}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("19:3-19:31", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + 2}; + int unint[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kOptionsFieldNumber, + OneofOptions::kUninterpretedOptionFieldNumber, + 2}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("20:3-20:31", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Enum value options + { + // option w/ message type that directly sets field + int path[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kValueFieldNumber, + 0, + EnumValueDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + kA_aFieldNumber}; + int unint[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kValueFieldNumber, + 0, + EnumValueDescriptorProto::kOptionsFieldNumber, + EnumValueOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 7); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("21:14-21:40", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kValueFieldNumber, + 1, + EnumValueDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kEnumTypeFieldNumber, + 0, + EnumDescriptorProto::kValueFieldNumber, + 1, + EnumValueDescriptorProto::kOptionsFieldNumber, + EnumValueOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("22:14-22:42", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Service option, repeated options + { + int path[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + 0}; + int unint[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kOptionsFieldNumber, + ServiceOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("26:3-26:35", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + 1}; + int unint[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kOptionsFieldNumber, + ServiceOptions::kUninterpretedOptionFieldNumber, + 1}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("27:3-27:35", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber, + 2}; + int unint[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kOptionsFieldNumber, + ServiceOptions::kUninterpretedOptionFieldNumber, + 2}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("28:3-28:35", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Method options + { + int path[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kMethodFieldNumber, + 1, + MethodDescriptorProto::kOptionsFieldNumber, + MethodOptions::kDeprecatedFieldNumber}; + int unint[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kMethodFieldNumber, + 1, + MethodDescriptorProto::kOptionsFieldNumber, + MethodOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("32:5-32:30", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kMethodFieldNumber, + 1, + MethodDescriptorProto::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kServiceFieldNumber, + 0, + ServiceDescriptorProto::kMethodFieldNumber, + 1, + MethodDescriptorProto::kOptionsFieldNumber, + MethodOptions::kUninterpretedOptionFieldNumber, + 1}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("33:5-33:41", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Extension range options + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 1, + DescriptorProto::kExtensionRangeFieldNumber, + 0, + DescriptorProto_ExtensionRange::kOptionsFieldNumber}; + std::vector vpath(path, path + 5); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("37:40-37:67", PrintSourceLocation(loc)); + } + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 1, + DescriptorProto::kExtensionRangeFieldNumber, + 0, + DescriptorProto_ExtensionRange::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 1, + DescriptorProto::kExtensionRangeFieldNumber, + 0, + DescriptorProto_ExtensionRange::kOptionsFieldNumber, + ExtensionRangeOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("37:41-37:66", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + { + int path[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 1, + DescriptorProto::kExtensionRangeFieldNumber, + 1, + DescriptorProto_ExtensionRange::kOptionsFieldNumber, + kCustomOptionFieldNumber}; + int unint[] = {FileDescriptorProto::kMessageTypeFieldNumber, + 1, + DescriptorProto::kExtensionRangeFieldNumber, + 1, + DescriptorProto_ExtensionRange::kOptionsFieldNumber, + ExtensionRangeOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 6); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("37:41-37:66", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 7); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } + + // Field option on extension + { + int path[] = {FileDescriptorProto::kExtensionFieldNumber, + 0, + FieldDescriptorProto::kOptionsFieldNumber, + FieldOptions::kPackedFieldNumber}; + int unint[] = {FileDescriptorProto::kExtensionFieldNumber, + 0, + FieldDescriptorProto::kOptionsFieldNumber, + FieldOptions::kUninterpretedOptionFieldNumber, + 0}; + std::vector vpath(path, path + 4); + EXPECT_TRUE(file_desc->GetSourceLocation(vpath, &loc)); + EXPECT_EQ("40:42-40:53", PrintSourceLocation(loc)); + + std::vector vunint(unint, unint + 5); + EXPECT_FALSE(file_desc->GetSourceLocation(vunint, &loc)); + } } // Missing SourceCodeInfo doesn't cause crash: