Improve pinning assembly when using a clang that supports it.

The new technique adds the relocation without adding an instruction that has to
be executed in the code path.

PiperOrigin-RevId: 613368436
pull/16061/head
Protobuf Team Bot 1 year ago committed by Copybara-Service
parent 9271acec79
commit 8a15c73d2c
  1. 4
      src/google/protobuf/compiler/cpp/field.cc
  2. 5
      src/google/protobuf/compiler/cpp/field_generators/message_field.cc
  3. 11
      src/google/protobuf/compiler/cpp/file.cc
  4. 7
      src/google/protobuf/compiler/cpp/helpers.cc
  5. 6
      src/google/protobuf/compiler/cpp/helpers.h
  6. 24
      src/google/protobuf/compiler/cpp/message.cc
  7. 4
      src/google/protobuf/compiler/java/java_features.pb.h
  8. 16
      src/google/protobuf/compiler/plugin.pb.h
  9. 4
      src/google/protobuf/cpp_features.pb.h
  10. 128
      src/google/protobuf/descriptor.pb.h
  11. 3
      src/google/protobuf/generated_message_util.cc
  12. 6
      src/google/protobuf/message.h
  13. 20
      src/google/protobuf/message_lite.h
  14. 28
      src/google/protobuf/port.h

@ -87,8 +87,8 @@ std::vector<Sub> FieldVars(const FieldDescriptor* field, const Options& opts) {
{"deprecated_attr", DeprecatedAttribute(opts, field)},
Sub("WeakDescriptorSelfPin",
UsingImplicitWeakDescriptor(field->file(), opts)
? absl::StrCat("::", ProtobufNamespace(opts),
"::internal::StrongReference(default_instance());")
? absl::StrCat(
StrongReferenceToType(field->containing_type(), opts), ";")
: "")
.WithSuffix(";"),
};

@ -72,9 +72,8 @@ std::vector<Sub> Vars(const FieldDescriptor* field, const Options& opts,
{"_weak", weak ? "_weak" : ""},
Sub("StrongRef",
!weak ? ""
: absl::Substitute("::google::protobuf::internal::StrongReference("
"reinterpret_cast<const $0&>($1));\n",
qualified_type, default_ref))
: absl::StrCat(
StrongReferenceToType(field->message_type(), opts), ";"))
.WithSuffix(";"),
};
}

@ -1266,13 +1266,10 @@ void FileGenerator::GenerateReflectionInitializationCode(io::Printer* p) {
if (UsingImplicitWeakDescriptor(file_, options_)) {
for (auto* pinned : GetMessagesToPinGloballyForWeakDescriptors(file_)) {
static_initializers_.push_back([this, pinned](auto* p) {
p->Emit(
{
{"default", QualifiedDefaultInstanceName(pinned, options_)},
},
R"cc(
::_pbi::StrongPointer(&$default$),
)cc");
p->Emit({{"pin", StrongReferenceToType(pinned, options_)}},
R"cc(
$pin$,
)cc");
});
}
}

@ -1495,6 +1495,13 @@ bool UsingImplicitWeakDescriptor(const FileDescriptor* file,
!options.opensource_runtime;
}
std::string StrongReferenceToType(const Descriptor* desc,
const Options& options) {
const auto name = QualifiedDefaultInstanceName(desc, options);
return absl::StrFormat("::%s::internal::StrongPointer<decltype(%s)*, &%s>()",
ProtobufNamespace(options), name, name);
}
std::string WeakDescriptorDataSection(absl::string_view prefix,
const Descriptor* descriptor,
int index_in_file_messages,

@ -786,7 +786,11 @@ void ListAllTypesForServices(const FileDescriptor* fd,
bool UsingImplicitWeakDescriptor(const FileDescriptor* file,
const Options& options);
// Generate the section name to be used for a data object when using implicit
// Generates a strong reference to the message in `desc`, as a statement.
std::string StrongReferenceToType(const Descriptor* desc,
const Options& options);
// Generates the section name to be used for a data object when using implicit
// weak descriptors. The prefix determines the kind of object and the section it
// will be merged into afterwards.
// See `UsingImplicitWeakDescriptor` above.

@ -489,8 +489,7 @@ std::vector<Sub> ClassVars(const Descriptor* desc, Options opts) {
Sub("WeakDescriptorSelfPin",
UsingImplicitWeakDescriptor(desc->file(), opts)
? absl::StrCat("::", ProtobufNamespace(opts),
"::internal::StrongReference(default_instance());")
? absl::StrCat(StrongReferenceToType(desc, opts), ";")
: "")
.WithSuffix(";"),
};
@ -2029,6 +2028,10 @@ void MessageGenerator::GenerateClassDefinition(io::Printer* p) {
$decl_oneof_has$;
$decl_data$;
$post_loop_handler$;
static constexpr const void* _raw_default_instance_ =
&_$classname$_default_instance_;
friend class ::$proto_ns$::MessageLite;
friend class ::$proto_ns$::Arena;
template <typename T>
@ -3526,9 +3529,10 @@ void MessageGenerator::GenerateClassData(io::Printer* p) {
const auto pin_weak_descriptor = [&] {
if (!UsingImplicitWeakDescriptor(descriptor_->file(), options_)) return;
p->Emit(R"cc(
::_pbi::StrongPointer(&_$classname$_default_instance_);
)cc");
p->Emit({{"pin", StrongReferenceToType(descriptor_, options_)}},
R"cc(
$pin$;
)cc");
// For CODE_SIZE types, we need to pin the submessages too.
// SPEED types will pin them via the TcParse table automatically.
@ -3536,11 +3540,11 @@ void MessageGenerator::GenerateClassData(io::Printer* p) {
for (int i = 0; i < descriptor_->field_count(); ++i) {
auto* field = descriptor_->field(i);
if (field->type() != field->TYPE_MESSAGE) continue;
p->Emit({{"sub_default_name", QualifiedDefaultInstanceName(
field->message_type(), options_)}},
R"cc(
::_pbi::StrongPointer(&$sub_default_name$);
)cc");
p->Emit(
{{"pin", StrongReferenceToType(field->message_type(), options_)}},
R"cc(
$pin$;
)cc");
}
};
p->Emit(

@ -274,6 +274,10 @@ class PROTOC_EXPORT JavaFeatures final : public ::google::protobuf::Message
1, 2, 1,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_JavaFeatures_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>

@ -301,6 +301,10 @@ class PROTOC_EXPORT Version final : public ::google::protobuf::Message
2, 4, 0,
47, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_Version_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -530,6 +534,10 @@ class PROTOC_EXPORT CodeGeneratorResponse_File final : public ::google::protobuf
2, 4, 1,
86, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_CodeGeneratorResponse_File_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -782,6 +790,10 @@ class PROTOC_EXPORT CodeGeneratorResponse final : public ::google::protobuf::Mes
3, 5, 1,
60, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_CodeGeneratorResponse_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -1041,6 +1053,10 @@ class PROTOC_EXPORT CodeGeneratorRequest final : public ::google::protobuf::Mess
3, 5, 3,
79, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_CodeGeneratorRequest_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>

@ -276,6 +276,10 @@ class PROTOBUF_EXPORT CppFeatures final : public ::google::protobuf::Message
1, 2, 1,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_CppFeatures_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>

@ -859,6 +859,10 @@ class PROTOBUF_EXPORT UninterpretedOption_NamePart final : public ::google::prot
1, 2, 0,
62, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_UninterpretedOption_NamePart_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -1119,6 +1123,10 @@ class PROTOBUF_EXPORT SourceCodeInfo_Location final : public ::google::protobuf:
3, 5, 0,
106, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_SourceCodeInfo_Location_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -1374,6 +1382,10 @@ class PROTOBUF_EXPORT GeneratedCodeInfo_Annotation final : public ::google::prot
3, 5, 1,
64, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_GeneratedCodeInfo_Annotation_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -1565,6 +1577,10 @@ class PROTOBUF_EXPORT FieldOptions_EditionDefault final : public ::google::proto
1, 2, 1,
57, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FieldOptions_EditionDefault_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -2094,6 +2110,10 @@ class PROTOBUF_EXPORT FeatureSet final : public ::google::protobuf::Message
3, 6, 6,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FeatureSet_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -2328,6 +2348,10 @@ class PROTOBUF_EXPORT ExtensionRangeOptions_Declaration final : public ::google:
3, 5, 0,
71, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_ExtensionRangeOptions_Declaration_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -2512,6 +2536,10 @@ class PROTOBUF_EXPORT EnumDescriptorProto_EnumReservedRange final : public ::goo
1, 2, 0,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_EnumDescriptorProto_EnumReservedRange_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -2693,6 +2721,10 @@ class PROTOBUF_EXPORT DescriptorProto_ReservedRange final : public ::google::pro
1, 2, 0,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_DescriptorProto_ReservedRange_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -2959,6 +2991,10 @@ class PROTOBUF_EXPORT UninterpretedOption final : public ::google::protobuf::Mes
3, 7, 1,
75, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_UninterpretedOption_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -3140,6 +3176,10 @@ class PROTOBUF_EXPORT SourceCodeInfo final : public ::google::protobuf::Message
0, 1, 1,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_SourceCodeInfo_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -3314,6 +3354,10 @@ class PROTOBUF_EXPORT GeneratedCodeInfo final : public ::google::protobuf::Messa
0, 1, 1,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_GeneratedCodeInfo_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -3497,6 +3541,10 @@ class PROTOBUF_EXPORT FeatureSetDefaults_FeatureSetEditionDefault final : public
1, 2, 2,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FeatureSetDefaults_FeatureSetEditionDefault_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -3879,6 +3927,10 @@ class PROTOBUF_EXPORT ServiceOptions final : public ::google::protobuf::Message
2, 3, 2,
0, 12>
_table_;
static constexpr const void* _raw_default_instance_ =
&_ServiceOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -4251,6 +4303,10 @@ class PROTOBUF_EXPORT OneofOptions final : public ::google::protobuf::Message
2, 2, 2,
0, 7>
_table_;
static constexpr const void* _raw_default_instance_ =
&_OneofOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -4666,6 +4722,10 @@ class PROTOBUF_EXPORT MethodOptions final : public ::google::protobuf::Message
3, 4, 3,
0, 12>
_table_;
static constexpr const void* _raw_default_instance_ =
&_MethodOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -5099,6 +5159,10 @@ class PROTOBUF_EXPORT MessageOptions final : public ::google::protobuf::Message
3, 7, 2,
0, 7>
_table_;
static constexpr const void* _raw_default_instance_ =
&_MessageOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -5783,6 +5847,10 @@ class PROTOBUF_EXPORT FileOptions final : public ::google::protobuf::Message
5, 21, 3,
202, 12>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FileOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -6407,6 +6475,10 @@ class PROTOBUF_EXPORT FieldOptions final : public ::google::protobuf::Message
4, 13, 7,
0, 7>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FieldOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -6619,6 +6691,10 @@ class PROTOBUF_EXPORT FeatureSetDefaults final : public ::google::protobuf::Mess
1, 3, 3,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FeatureSetDefaults_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -7040,6 +7116,10 @@ class PROTOBUF_EXPORT ExtensionRangeOptions final : public ::google::protobuf::M
3, 4, 4,
0, 12>
_table_;
static constexpr const void* _raw_default_instance_ =
&_ExtensionRangeOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -7437,6 +7517,10 @@ class PROTOBUF_EXPORT EnumValueOptions final : public ::google::protobuf::Messag
3, 4, 2,
0, 7>
_table_;
static constexpr const void* _raw_default_instance_ =
&_EnumValueOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -7846,6 +7930,10 @@ class PROTOBUF_EXPORT EnumOptions final : public ::google::protobuf::Message
3, 5, 2,
0, 7>
_table_;
static constexpr const void* _raw_default_instance_ =
&_EnumOptions_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -8041,6 +8129,10 @@ class PROTOBUF_EXPORT OneofDescriptorProto final : public ::google::protobuf::Me
1, 2, 1,
49, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_OneofDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -8292,6 +8384,10 @@ class PROTOBUF_EXPORT MethodDescriptorProto final : public ::google::protobuf::M
3, 6, 1,
71, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_MethodDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -8674,6 +8770,10 @@ class PROTOBUF_EXPORT FieldDescriptorProto final : public ::google::protobuf::Me
4, 11, 3,
96, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FieldDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -8886,6 +8986,10 @@ class PROTOBUF_EXPORT EnumValueDescriptorProto final : public ::google::protobuf
2, 3, 1,
53, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_EnumValueDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -9084,6 +9188,10 @@ class PROTOBUF_EXPORT DescriptorProto_ExtensionRange final : public ::google::pr
2, 3, 1,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_DescriptorProto_ExtensionRange_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -9294,6 +9402,10 @@ class PROTOBUF_EXPORT ServiceDescriptorProto final : public ::google::protobuf::
2, 3, 2,
51, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_ServiceDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -9552,6 +9664,10 @@ class PROTOBUF_EXPORT EnumDescriptorProto final : public ::google::protobuf::Mes
3, 5, 3,
61, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_EnumDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -9903,6 +10019,10 @@ class PROTOBUF_EXPORT DescriptorProto final : public ::google::protobuf::Message
4, 10, 8,
65, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_DescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -10305,6 +10425,10 @@ class PROTOBUF_EXPORT FileDescriptorProto final : public ::google::protobuf::Mes
4, 13, 7,
79, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FileDescriptorProto_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>
@ -10491,6 +10615,10 @@ class PROTOBUF_EXPORT FileDescriptorSet final : public ::google::protobuf::Messa
0, 1, 1,
0, 2>
_table_;
static constexpr const void* _raw_default_instance_ =
&_FileDescriptorSet_default_instance_;
friend class ::google::protobuf::MessageLite;
friend class ::google::protobuf::Arena;
template <typename T>

@ -69,7 +69,8 @@ extern const char __start_pb_defaults;
extern const char __stop_pb_defaults;
}
static void InitWeakDefaults() {
StrongPointer(&dummy_weak_default); // force link the dummy entry.
// force link the dummy entry.
StrongPointer<DummyWeakDefault*, &dummy_weak_default>();
// We don't know the size of each object, but we know the layout of the tail.
// It contains a WeakDescriptorDefaultTail object.
// As such, we iterate the section backwards.

@ -1418,7 +1418,7 @@ const T* DynamicCastToGenerated(const Message* from) {
(void)unused;
#if PROTOBUF_RTTI
internal::StrongReference(T::default_instance());
internal::StrongReferenceToType<T>();
return dynamic_cast<const T*>(from);
#else
bool ok = from != nullptr &&
@ -1457,7 +1457,7 @@ T& DynamicCastToGenerated(Message& from) {
// instance T and T is a type derived from base Message class.
template <typename T>
const T* DownCastToGenerated(const Message* from) {
internal::StrongReference(T::default_instance());
internal::StrongReferenceToType<T>();
ABSL_DCHECK(DynamicCastToGenerated<T>(from) == from)
<< "Cannot downcast " << from->GetTypeName() << " to "
<< T::default_instance().GetTypeName();
@ -1505,7 +1505,7 @@ T& DownCastToGenerated(Message& from) {
// of loops (on x86-64 it compiles into two "mov" instructions).
template <typename T>
void LinkMessageReflection() {
internal::StrongReference(T::default_instance);
internal::StrongReferenceToType<T>();
}
// =============================================================================

@ -675,6 +675,26 @@ class PROTOBUF_EXPORT MessageLite {
void LogInitializationErrorMessage() const;
bool MergeFromImpl(io::CodedInputStream* input, ParseFlags parse_flags);
template <typename T, const void* ptr = T::_raw_default_instance_>
static constexpr auto GetStrongPointerForTypeImpl(int) {
return ptr;
}
template <typename T>
static constexpr auto GetStrongPointerForTypeImpl(char) {
return &T::default_instance;
}
// Return a pointer we can use to make a strong reference to a type.
// Ideally, this is a pointer to the default instance.
// If we can't get that, then we use a pointer to the `default_instance`
// function. The latter always works but pins the function artificially into
// the binary so we avoid it.
template <typename T>
static constexpr auto GetStrongPointerForType() {
return GetStrongPointerForTypeImpl<T>(0);
}
template <typename T>
friend void internal::StrongReferenceToType();
};
namespace internal {

@ -39,7 +39,7 @@ class MessageLite;
namespace internal {
template <typename T>
void StrongPointer(T* var) {
inline PROTOBUF_ALWAYS_INLINE void StrongPointer(T* var) {
#if defined(__GNUC__)
asm("" : : "r"(var));
#else
@ -48,6 +48,32 @@ void StrongPointer(T* var) {
#endif
}
// Similar to the overload above, but optimized for constant inputs.
template <typename T, T ptr>
inline PROTOBUF_ALWAYS_INLINE void StrongPointer() {
#if defined(__x86_64__) && defined(__linux__) && !defined(__APPLE__) && \
!defined(__ANDROID__) && defined(__clang__) && __clang_major__ >= 19 && \
!defined(PROTOBUF_INTERNAL_TEMPORARY_STRONG_POINTER_OPT_OUT)
// This injects a relocation in the code path without having to run code, but
// we can only do it with a newer clang.
asm(".reloc ., BFD_RELOC_NONE, %p0" ::"Ws"(ptr));
#else
StrongPointer(ptr);
#endif
}
template <typename T>
inline PROTOBUF_ALWAYS_INLINE void StrongReferenceToType() {
constexpr auto ptr = T::template GetStrongPointerForType<T>();
#if defined(__cpp_nontype_template_args) && \
__cpp_nontype_template_args >= 201411L
// We can only use `ptr` as a template parameter since C++17
return StrongPointer<decltype(ptr), ptr>();
#else
return StrongPointer(ptr);
#endif
}
// See comments on `AllocateAtLeast` for information on size returning new.
struct SizedPtr {

Loading…
Cancel
Save