diff --git a/python/build_targets.bzl b/python/build_targets.bzl index 0a20662c3c..034fddf481 100644 --- a/python/build_targets.bzl +++ b/python/build_targets.bzl @@ -239,6 +239,28 @@ def build_targets(name): strip_prefix = "src", ) + internal_copy_files( + name = "copied_test_dependency_proto_files", + srcs = [ + "//src/google/protobuf:cpp_features_proto_srcs", + ], + strip_prefix = "src", + ) + + internal_py_proto_library( + name = "test_dependency_proto_py_pb2", + srcs = [":copied_test_dependency_proto_files"], + include = ".", + default_runtime = "", + protoc = "//:protoc", + srcs_version = "PY2AND3", + visibility = [ + "//:__pkg__", + "//upb:__subpackages__", + ], + deps = [":well_known_types_py_pb2"], + ) + internal_py_proto_library( name = "python_common_test_protos", testonly = 1, @@ -248,7 +270,7 @@ def build_targets(name): protoc = "//:protoc", srcs_version = "PY2AND3", visibility = ["//:__pkg__"], - deps = [":well_known_types_py_pb2"], + deps = [":well_known_types_py_pb2", ":test_dependency_proto_py_pb2"], ) internal_py_proto_library( diff --git a/src/google/protobuf/BUILD.bazel b/src/google/protobuf/BUILD.bazel index 5eb007e440..bc0f3e5345 100644 --- a/src/google/protobuf/BUILD.bazel +++ b/src/google/protobuf/BUILD.bazel @@ -688,7 +688,7 @@ filegroup( filegroup( name = "cpp_features_proto_srcs", srcs = ["cpp_features.proto"], - visibility = ["//src/google/protobuf/compiler/cpp:__pkg__"], + visibility = ["//:__subpackages__"], ) filegroup( @@ -777,6 +777,7 @@ filegroup( "unittest_no_field_presence.proto", "unittest_preserve_unknown_enum.proto", "unittest_preserve_unknown_enum2.proto", + "unittest_string_view.proto", ], visibility = ["//:__subpackages__"], ) @@ -798,6 +799,7 @@ proto_library( deps = [ ":any_proto", ":api_proto", + ":cpp_features_proto", ":descriptor_proto", ":duration_proto", ":empty_proto", @@ -854,6 +856,7 @@ proto_library( deps = [ ":any_proto", ":api_proto", + ":cpp_features_proto", ":descriptor_proto", ":duration_proto", ":empty_proto", @@ -927,6 +930,30 @@ cc_proto_library( deps = [":test_protos"], ) +proto_library( + name = "unittest_string_view_proto", + srcs = ["unittest_string_view.proto"], + strip_import_prefix = "/src", + deps = [":cpp_features_proto"], +) + +cc_proto_library( + name = "unittest_string_view_cc_proto", + deps = [":unittest_string_view_proto"], +) + +cc_test( + name = "string_view_test", + srcs = ["string_view_test.cc"], + deps = [ + ":protobuf", + ":unittest_string_view_cc_proto", + "@com_google_absl//absl/strings:string_view", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + # Filegroup for golden comparison test: filegroup( name = "descriptor_cc_srcs", diff --git a/src/google/protobuf/compiler/cpp/BUILD.bazel b/src/google/protobuf/compiler/cpp/BUILD.bazel index f4a1ffeea1..0e540e982a 100644 --- a/src/google/protobuf/compiler/cpp/BUILD.bazel +++ b/src/google/protobuf/compiler/cpp/BUILD.bazel @@ -66,6 +66,7 @@ cc_library( "field_generators/message_field.cc", "field_generators/primitive_field.cc", "field_generators/string_field.cc", + "field_generators/string_view_field.cc", "file.cc", "generator.cc", "message.cc", diff --git a/src/google/protobuf/compiler/cpp/field.cc b/src/google/protobuf/compiler/cpp/field.cc index eedb5b64bd..91bb452363 100644 --- a/src/google/protobuf/compiler/cpp/field.cc +++ b/src/google/protobuf/compiler/cpp/field.cc @@ -17,6 +17,7 @@ #include #include +#include "absl/base/attributes.h" #include "absl/log/absl_check.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -24,9 +25,11 @@ #include "absl/types/optional.h" #include "absl/types/span.h" #include "google/protobuf/compiler/cpp/field_generators/generators.h" +#include "google/protobuf/compiler/cpp/generator.h" #include "google/protobuf/compiler/cpp/helpers.h" #include "google/protobuf/compiler/cpp/options.h" #include "google/protobuf/compiler/cpp/tracker.h" +#include "google/protobuf/cpp_features.pb.h" #include "google/protobuf/descriptor.h" #include "google/protobuf/descriptor.pb.h" #include "google/protobuf/io/printer.h" @@ -223,6 +226,40 @@ void FieldGeneratorBase::GenerateCopyConstructorCode(io::Printer* p) const { } namespace { +// Use internal types instead of ctype or string_type. +enum class StringType { + kView, + kString, + kCord, + kStringPiece, +}; + +StringType GetStringType(const FieldDescriptor& field) { + ABSL_CHECK_EQ(field.cpp_type(), FieldDescriptor::CPPTYPE_STRING); + + if (field.options().has_ctype()) { + switch (field.options().ctype()) { + case FieldOptions::CORD: + return StringType::kCord; + case FieldOptions::STRING_PIECE: + return StringType::kStringPiece; + default: + return StringType::kString; + } + } + + const pb::CppFeatures& cpp_features = + CppGenerator::GetResolvedSourceFeatures(field).GetExtension(::pb::cpp); + switch (cpp_features.string_type()) { + case pb::CppFeatures::CORD: + return StringType::kCord; + case pb::CppFeatures::VIEW: + return StringType::kView; + default: + return StringType::kString; + } +} + std::unique_ptr MakeGenerator(const FieldDescriptor* field, const Options& options, MessageSCCAnalyzer* scc) { @@ -236,8 +273,13 @@ std::unique_ptr MakeGenerator(const FieldDescriptor* field, switch (field->cpp_type()) { case FieldDescriptor::CPPTYPE_MESSAGE: return MakeRepeatedMessageGenerator(field, options, scc); - case FieldDescriptor::CPPTYPE_STRING: - return MakeRepeatedStringGenerator(field, options, scc); + case FieldDescriptor::CPPTYPE_STRING: { + if (GetStringType(*field) == StringType::kView) { + return MakeRepeatedStringViewGenerator(field, options, scc); + } else { + return MakeRepeatedStringGenerator(field, options, scc); + } + } case FieldDescriptor::CPPTYPE_ENUM: return MakeRepeatedEnumGenerator(field, options, scc); default: @@ -253,19 +295,25 @@ std::unique_ptr MakeGenerator(const FieldDescriptor* field, switch (field->cpp_type()) { case FieldDescriptor::CPPTYPE_MESSAGE: return MakeSinguarMessageGenerator(field, options, scc); - case FieldDescriptor::CPPTYPE_STRING: - if (field->type() == FieldDescriptor::TYPE_BYTES && - field->options().ctype() == FieldOptions::CORD) { - if (field->real_containing_oneof()) { - return MakeOneofCordGenerator(field, options, scc); - } else { - return MakeSingularCordGenerator(field, options, scc); - } - } else { - return MakeSinguarStringGenerator(field, options, scc); - } case FieldDescriptor::CPPTYPE_ENUM: return MakeSinguarEnumGenerator(field, options, scc); + case FieldDescriptor::CPPTYPE_STRING: { + switch (GetStringType(*field)) { + case StringType::kView: + return MakeSingularStringViewGenerator(field, options, scc); + case StringType::kCord: + if (field->type() == FieldDescriptor::TYPE_BYTES) { + if (field->real_containing_oneof()) { + return MakeOneofCordGenerator(field, options, scc); + } else { + return MakeSingularCordGenerator(field, options, scc); + } + } + ABSL_FALLTHROUGH_INTENDED; + default: + return MakeSinguarStringGenerator(field, options, scc); + } + } default: return MakeSinguarPrimitiveGenerator(field, options, scc); } diff --git a/src/google/protobuf/compiler/cpp/field_generators/generators.h b/src/google/protobuf/compiler/cpp/field_generators/generators.h index 28948ec350..69927b0741 100644 --- a/src/google/protobuf/compiler/cpp/field_generators/generators.h +++ b/src/google/protobuf/compiler/cpp/field_generators/generators.h @@ -54,6 +54,14 @@ std::unique_ptr MakeRepeatedStringGenerator( const FieldDescriptor* desc, const Options& options, MessageSCCAnalyzer* scc); +std::unique_ptr MakeSingularStringViewGenerator( + const FieldDescriptor* desc, const Options& options, + MessageSCCAnalyzer* scc); + +std::unique_ptr MakeRepeatedStringViewGenerator( + const FieldDescriptor* desc, const Options& options, + MessageSCCAnalyzer* scc); + std::unique_ptr MakeSinguarMessageGenerator( const FieldDescriptor* desc, const Options& options, MessageSCCAnalyzer* scc); diff --git a/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc b/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc new file mode 100644 index 0000000000..fe801a4ded --- /dev/null +++ b/src/google/protobuf/compiler/cpp/field_generators/string_view_field.cc @@ -0,0 +1,846 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. + +#include +#include +#include + +#include "absl/log/absl_check.h" +#include "absl/memory/memory.h" +#include "absl/strings/str_cat.h" +#include "google/protobuf/compiler/cpp/field.h" +#include "google/protobuf/compiler/cpp/helpers.h" +#include "google/protobuf/compiler/cpp/options.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/descriptor.pb.h" +#include "google/protobuf/io/printer.h" + +namespace google { +namespace protobuf { +namespace compiler { +namespace cpp { +namespace { +using ::google::protobuf::internal::cpp::HasHasbit; +using ::google::protobuf::io::AnnotationCollector; +using Sub = ::google::protobuf::io::Printer::Sub; + +std::vector Vars(const FieldDescriptor* field, const Options& opts) { + auto trivial_default = + absl::StrCat("::", ProtobufNamespace(opts), + "::internal::GetEmptyStringAlreadyInited()"); + auto lazy_var = + absl::StrCat(QualifiedClassName(field->containing_type(), opts), + "::", MakeDefaultFieldName(field)); + + bool empty_default = field->default_value_string().empty(); + bool bytes = field->type() == FieldDescriptor::TYPE_BYTES; + + return { + {"kDefault", DefaultValue(opts, field)}, + {"kDefaultLen", field->default_value_string().size()}, + {"default_variable_name", MakeDefaultName(field)}, + {"default_variable_field", MakeDefaultFieldName(field)}, + + {"kDefaultStr", + !empty_default ? absl::StrCat(lazy_var, ".get()") : trivial_default}, + {"kDefaultValue", + !empty_default ? "nullptr" : absl::StrCat("&", trivial_default)}, + + {"lazy_var", lazy_var}, + Sub{"lazy_args", !empty_default ? absl::StrCat(lazy_var, ",") : ""} + .WithSuffix(","), + + {"byte", bytes ? "void" : "char"}, + {"Set", bytes ? "SetBytes" : "Set"}, + }; +} + +class SingularStringView : public FieldGeneratorBase { + public: + SingularStringView(const FieldDescriptor* field, const Options& opts, + MessageSCCAnalyzer* scc) + : FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {} + ~SingularStringView() override = default; + + std::vector MakeVars() const override { return Vars(field_, *opts_); } + + bool IsInlined() const override { return is_inlined(); } + + ArenaDtorNeeds NeedsArenaDestructor() const override { + return is_inlined() ? ArenaDtorNeeds::kOnDemand : ArenaDtorNeeds::kNone; + } + + void GeneratePrivateMembers(io::Printer* p) const override { + // Skips the automatic destruction if inlined; rather calls it explicitly if + // allocating arena is null. + p->Emit({{"Str", is_inlined() ? "InlinedStringField" : "ArenaStringPtr"}}, + R"cc( + $pbi$::$Str$ $name$_; + )cc"); + } + + bool RequiresArena(GeneratorFunction function) const override { + switch (function) { + case GeneratorFunction::kMergeFrom: + return is_oneof(); + } + return false; + } + + void GenerateMergingCode(io::Printer* p) const override { + if (is_oneof()) { + p->Emit(R"cc( + if (oneof_needs_init) { + _this->$field_$.InitDefault(); + } + _this->$field_$.Set(from._internal_$name$(), arena); + )cc"); + } else { + p->Emit(R"cc( + _this->_internal_set_$name$(from._internal_$name$()); + )cc"); + } + } + + void GenerateArenaDestructorCode(io::Printer* p) const override { + if (!is_inlined()) return; + + p->Emit(R"cc( + if (!_this->_internal_$name$_donated()) { + _this->$field_$.~InlinedStringField(); + } + )cc"); + } + + void GenerateNonInlineAccessorDefinitions(io::Printer* p) const override { + if (EmptyDefault()) return; + p->Emit(R"cc( + /*static*/ const ::_pbi::LazyString $Msg$::$default_variable_field${ + {{$kDefault$, $kDefaultLen$}}, + {nullptr}, + }; + )cc"); + } + + void GenerateByteSize(io::Printer* p) const override { + p->Emit(R"cc( + total_size += $kTagBytes$ + $pbi$::WireFormatLite::$DeclaredType$Size( + this->_internal_$name$()); + )cc"); + } + + void GenerateCopyAggregateInitializer(io::Printer* p) const override { + p->Emit(R"cc( + decltype($field_$){}, + )cc"); + } + + void GenerateMemberConstexprConstructor(io::Printer* p) const override { + if (is_inlined()) { + p->Emit("$name$_(nullptr, false)"); + } else { + p->Emit( + "$name$_(\n" + " &$pbi$::fixed_address_empty_string,\n" + " ::_pbi::ConstantInitialized())"); + } + } + + void GenerateMemberConstructor(io::Printer* p) const override { + if (is_inlined()) { + p->Emit("$name$_{}"); + } else if (EmptyDefault()) { + p->Emit("$name$_(arena)"); + } else { + p->Emit("$name$_(arena, $default_variable_field$)"); + } + } + + void GenerateMemberCopyConstructor(io::Printer* p) const override { + if (is_inlined() || EmptyDefault()) { + p->Emit("$name$_(arena, from.$name$_)"); + } else { + p->Emit("$name$_(arena, from.$name$_, $default_variable_name$)"); + } + } + + void GenerateOneofCopyConstruct(io::Printer* p) const override { + if (is_inlined() || EmptyDefault()) { + p->Emit("new (&$field$) decltype($field$){arena, from.$field$};\n"); + } else { + p->Emit( + "new (&$field$) decltype($field$){arena, from.$field$," + " $default_variable_field$};\n"); + } + } + + void GenerateStaticMembers(io::Printer* p) const override; + void GenerateAccessorDeclarations(io::Printer* p) const override; + void GenerateInlineAccessorDefinitions(io::Printer* p) const override; + void GenerateClearingCode(io::Printer* p) const override; + void GenerateMessageClearingCode(io::Printer* p) const override; + void GenerateSwappingCode(io::Printer* p) const override; + void GenerateConstructorCode(io::Printer* p) const override; + void GenerateCopyConstructorCode(io::Printer* p) const override; + void GenerateDestructorCode(io::Printer* p) const override; + void GenerateSerializeWithCachedSizesToArray(io::Printer* p) const override; + void GenerateConstexprAggregateInitializer(io::Printer* p) const override; + void GenerateAggregateInitializer(io::Printer* p) const override; + + private: + bool EmptyDefault() const { return field_->default_value_string().empty(); } + + const FieldDescriptor* field_; + const Options* opts_; +}; + +void SingularStringView::GenerateStaticMembers(io::Printer* p) const { + if (!EmptyDefault()) { + p->Emit(R"cc( + static const $pbi$::LazyString $default_variable_name$; + )cc"); + } + if (is_inlined()) { + // `_init_inline_xxx` is used for initializing default instances. + p->Emit(R"cc( + static std::true_type _init_inline_$name$_; + )cc"); + } +} + +void SingularStringView::GenerateAccessorDeclarations(io::Printer* p) const { + ABSL_CHECK(!field_->options().has_ctype()); + + auto vars = AnnotatedAccessors(field_, {"", "set_allocated_"}); + vars.push_back(Sub{ + "release_name", + SafeFunctionName(field_->containing_type(), field_, "release_"), + } + .AnnotatedAs(field_)); + auto v1 = p->WithVars(vars); + auto v2 = p->WithVars( + AnnotatedAccessors(field_, {"set_"}, AnnotationCollector::kSet)); + auto v3 = p->WithVars( + AnnotatedAccessors(field_, {"mutable_"}, AnnotationCollector::kAlias)); + + p->Emit( + {{"donated", + [&] { + if (!is_inlined()) return; + p->Emit(R"cc( + inline PROTOBUF_ALWAYS_INLINE bool _internal_$name$_donated() const; + )cc"); + }}}, + R"cc( + $DEPRECATED$ absl::string_view $name$() const; + template + $DEPRECATED$ void $set_name$(Arg_&& arg); + + private: + const std::string& _internal_$name$() const; + inline PROTOBUF_ALWAYS_INLINE void _internal_set_$name$( + absl::string_view value); + $donated$; + + public: + )cc"); +} + +void UpdateHasbitSet(io::Printer* p, bool is_oneof) { + if (!is_oneof) { + p->Emit(R"cc( + $set_hasbit$; + )cc"); + return; + } + + p->Emit(R"cc( + if ($not_has_field$) { + clear_$oneof_name$(); + + set_has_$name_internal$(); + $field_$.InitDefault(); + } + )cc"); +} + +void ArgsForSetter(io::Printer* p, bool inlined) { + if (!inlined) { + p->Emit("GetArena()"); + return; + } + p->Emit( + "GetArena(), _internal_$name_internal$_donated(), " + "&$donating_states_word$, $mask_for_undonate$, this"); +} + +void SingularStringView::GenerateInlineAccessorDefinitions( + io::Printer* p) const { + p->Emit( + { + {"if_IsDefault", + [&] { + if (EmptyDefault() || is_oneof()) return; + p->Emit(R"cc( + if ($field_$.IsDefault()) { + return $default_variable_field$.get(); + } + )cc"); + }}, + {"update_hasbit", [&] { UpdateHasbitSet(p, is_oneof()); }}, + {"set_args", [&] { ArgsForSetter(p, is_inlined()); }}, + {"check_hasbit", + [&] { + if (!is_oneof()) return; + p->Emit(R"cc( + if ($not_has_field$) { + return $kDefaultStr$; + } + )cc"); + }}, + {"release_name", + SafeFunctionName(field_->containing_type(), field_, "release_")}, + }, + R"cc( + inline absl::string_view $Msg$::$name$() const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + $WeakDescriptorSelfPin$; + $annotate_get$; + // @@protoc_insertion_point(field_get:$pkg.Msg.field$) + $if_IsDefault$; + return _internal_$name_internal$(); + } + template + inline PROTOBUF_ALWAYS_INLINE void $Msg$::set_$name$(Arg_&& arg) { + $WeakDescriptorSelfPin$; + $TsanDetectConcurrentMutation$; + $PrepareSplitMessageForWrite$; + $update_hasbit$; + $field_$.$Set$(static_cast(arg), $set_args$); + $annotate_set$; + // @@protoc_insertion_point(field_set:$pkg.Msg.field$) + } + inline const std::string& $Msg$::_internal_$name_internal$() const { + $TsanDetectConcurrentRead$; + $check_hasbit$; + return $field_$.Get(); + } + inline void $Msg$::_internal_set_$name_internal$(absl::string_view value) { + $TsanDetectConcurrentMutation$; + $update_hasbit$; + //~ Don't use $Set$ here; we always want the std::string variant + //~ regardless of whether this is a `bytes` field. + $field_$.Set(value, $set_args$); + } + )cc"); + + if (is_inlined()) { + p->Emit(R"cc( + inline bool $Msg$::_internal_$name_internal$_donated() const { + return $inlined_string_donated$; + } + )cc"); + } +} + +void SingularStringView::GenerateClearingCode(io::Printer* p) const { + if (is_oneof()) { + p->Emit(R"cc( + $field_$.Destroy(); + )cc"); + return; + } + + if (EmptyDefault()) { + p->Emit(R"cc( + $field_$.ClearToEmpty(); + )cc"); + return; + } + + ABSL_DCHECK(!is_inlined()); + p->Emit(R"cc( + $field_$.ClearToDefault($lazy_var$, GetArena()); + )cc"); +} + +void SingularStringView::GenerateMessageClearingCode(io::Printer* p) const { + if (is_oneof()) { + p->Emit(R"cc( + $field_$.Destroy(); + )cc"); + return; + } + + // Two-dimension specialization here: supporting arenas, field presence, or + // not, and default value is the empty string or not. Complexity here ensures + // the minimal number of branches / amount of extraneous code at runtime + // (given that the below methods are inlined one-liners)! + + // If we have a hasbit, then the Clear() method of the protocol buffer + // will have checked that this field is set. If so, we can avoid redundant + // checks against the default variable. + + if (is_inlined() && HasHasbit(field_)) { + p->Emit(R"cc( + $DCHK$(!$field_$.IsDefault()); + )cc"); + } + + if (!EmptyDefault()) { + // Clear to a non-empty default is more involved, as we try to use the + // Arena if one is present and may need to reallocate the string. + p->Emit(R"cc( + $field_$.ClearToDefault($lazy_var$, GetArena()); + )cc"); + return; + } + + p->Emit({{"Clear", + HasHasbit(field_) ? "ClearNonDefaultToEmpty" : "ClearToEmpty"}}, + R"cc( + $field_$.$Clear$(); + )cc"); +} + +void SingularStringView::GenerateSwappingCode(io::Printer* p) const { + if (is_oneof()) { + // Don't print any swapping code. Swapping the union will swap this field. + return; + } + + if (!is_inlined()) { + p->Emit(R"cc( + ::_pbi::ArenaStringPtr::InternalSwap(&$field_$, &other->$field_$, arena); + )cc"); + return; + } + + p->Emit(R"cc( + { + bool lhs_dtor_registered = ($inlined_string_donated_array$[0] & 1) == 0; + bool rhs_dtor_registered = + (other->$inlined_string_donated_array$[0] & 1) == 0; + ::_pbi::InlinedStringField::InternalSwap( + &$field_$, lhs_dtor_registered, this, &other->$field_$, + rhs_dtor_registered, other, arena); + } + )cc"); +} + +void SingularStringView::GenerateConstructorCode(io::Printer* p) const { + if ((is_inlined() && EmptyDefault()) || is_oneof()) return; + ABSL_DCHECK(!is_inlined()); + + p->Emit(R"cc( + $field_$.InitDefault(); + )cc"); + + if (EmptyDefault()) { + p->Emit(R"cc( +#ifdef PROTOBUF_FORCE_COPY_DEFAULT_STRING + $field_$.Set("", GetArena()); +#endif // PROTOBUF_FORCE_COPY_DEFAULT_STRING + )cc"); + } +} + +void SingularStringView::GenerateCopyConstructorCode(io::Printer* p) const { + GenerateConstructorCode(p); + + if (is_inlined()) { + p->Emit(R"cc( + new (&_this->$field_$)::_pbi::InlinedStringField; + )cc"); + } + + p->Emit( + {{"hazzer", + [&] { + if (HasHasbit(field_)) { + p->Emit(R"cc((from.$has_hasbit$) != 0)cc"); + } else { + p->Emit(R"cc(!from._internal_$name$().empty())cc"); + } + }}, + {"set_args", + [&] { + if (!is_inlined()) { + p->Emit("_this->GetArena()"); + } else { + p->Emit( + "_this->GetArena(), " + "_this->_internal_$name$_donated(), " + "&_this->$donating_states_word$, $mask_for_undonate$, _this"); + } + }}}, + R"cc( + if ($hazzer$) { + _this->$field_$.Set(from._internal_$name$(), $set_args$); + } + )cc"); +} + +void SingularStringView::GenerateDestructorCode(io::Printer* p) const { + if (is_inlined()) { + ABSL_DCHECK(!should_split()); + return; + } + + if (should_split()) { + p->Emit(R"cc( + $cached_split_ptr$->$name$_.Destroy(); + )cc"); + return; + } + + p->Emit(R"cc( + $field_$.Destroy(); + )cc"); +} + +void SingularStringView::GenerateSerializeWithCachedSizesToArray( + io::Printer* p) const { + p->Emit({{"utf8_check", + [&] { + GenerateUtf8CheckCodeForString(p, field_, options_, false, + "_s.data(), " + "static_cast(_s.length()),"); + }}}, + R"cc( + const std::string& _s = this->_internal_$name$(); + $utf8_check$; + target = stream->Write$DeclaredType$MaybeAliased($number$, _s, target); + )cc"); +} + +void SingularStringView::GenerateConstexprAggregateInitializer( + io::Printer* p) const { + if (is_inlined()) { + p->Emit(R"cc( + /*decltype($field_$)*/ {nullptr, false}, + )cc"); + } else { + p->Emit(R"cc( + /*decltype($field_$)*/ { + &::_pbi::fixed_address_empty_string, + ::_pbi::ConstantInitialized{}, + }, + )cc"); + } +} + +void SingularStringView::GenerateAggregateInitializer(io::Printer* p) const { + if (should_split()) { + ABSL_CHECK(!is_inlined()); + p->Emit(R"cc( + decltype(Impl_::Split::$name$_){}, + )cc"); + } else if (!is_inlined()) { + p->Emit(R"cc( + decltype($field_$){}, + )cc"); + } else { + p->Emit(R"cc( + decltype($field_$){arena}, + )cc"); + } +} + +class RepeatedStringView : public FieldGeneratorBase { + public: + RepeatedStringView(const FieldDescriptor* field, const Options& opts, + MessageSCCAnalyzer* scc) + : FieldGeneratorBase(field, opts, scc), field_(field), opts_(&opts) {} + ~RepeatedStringView() override = default; + + std::vector MakeVars() const override { return Vars(field_, *opts_); } + + void GeneratePrivateMembers(io::Printer* p) const override { + if (should_split()) { + p->Emit(R"cc( + $pbi$::RawPtr<$pb$::RepeatedPtrField> $name$_; + )cc"); + } else { + p->Emit(R"cc( + $pb$::RepeatedPtrField $name$_; + )cc"); + } + } + + void GenerateClearingCode(io::Printer* p) const override { + if (should_split()) { + p->Emit("$field_$.ClearIfNotDefault();\n"); + } else { + p->Emit("$field_$.Clear();\n"); + } + } + + void GenerateMergingCode(io::Printer* p) const override { + // TODO: experiment with simplifying this to be + // `if (!from.empty()) { body(); }` for both split and non-split cases. + auto body = [&] { + p->Emit(R"cc( + _this->_internal_mutable_$name$()->MergeFrom(from._internal_$name$()); + )cc"); + }; + if (!should_split()) { + body(); + } else { + p->Emit({{"body", body}}, R"cc( + if (!from.$field_$.IsDefault()) { + $body$; + } + )cc"); + } + } + + void GenerateSwappingCode(io::Printer* p) const override { + ABSL_CHECK(!should_split()); + p->Emit(R"cc( + $field_$.InternalSwap(&other->$field_$); + )cc"); + } + + void GenerateDestructorCode(io::Printer* p) const override { + if (should_split()) { + p->Emit(R"cc( + $field_$.DeleteIfNotDefault(); + )cc"); + } + } + + void GenerateConstructorCode(io::Printer* p) const override {} + + void GenerateCopyConstructorCode(io::Printer* p) const override { + if (should_split()) { + p->Emit(R"cc( + if (!from._internal_$name$().empty()) { + _internal_mutable_$name$()->MergeFrom(from._internal_$name$()); + } + )cc"); + } + } + + void GenerateByteSize(io::Printer* p) const override { + p->Emit(R"cc( + total_size += $kTagBytes$ * $pbi$::FromIntSize(_internal_$name$().size()); + for (int i = 0, n = _internal_$name$().size(); i < n; ++i) { + total_size += $pbi$::WireFormatLite::$DeclaredType$Size( + _internal_$name$().Get(i)); + } + )cc"); + } + + void GenerateAccessorDeclarations(io::Printer* p) const override; + void GenerateInlineAccessorDefinitions(io::Printer* p) const override; + void GenerateSerializeWithCachedSizesToArray(io::Printer* p) const override; + + private: + const FieldDescriptor* field_; + const Options* opts_; +}; + +void RepeatedStringView::GenerateAccessorDeclarations(io::Printer* p) const { + bool unknown_ctype = + field_->options().ctype() != internal::cpp::EffectiveStringCType(field_); + + if (unknown_ctype) { + p->Emit(R"cc( + private: // Hidden due to unknown ctype option. + )cc"); + } + + auto v1 = p->WithVars(AnnotatedAccessors(field_, {"", "_internal_"})); + auto v2 = p->WithVars( + AnnotatedAccessors(field_, {"set_", "add_"}, AnnotationCollector::kSet)); + auto v3 = p->WithVars( + AnnotatedAccessors(field_, {"mutable_"}, AnnotationCollector::kAlias)); + + p->Emit(R"cc( + $DEPRECATED$ absl::string_view $name$(int index) const; + $DEPRECATED$ void $set_name$(int index, const std::string& value); + $DEPRECATED$ void $set_name$(int index, std::string&& value); + $DEPRECATED$ void $set_name$(int index, const char* value); + $DEPRECATED$ void $set_name$(int index, absl::string_view value); + $DEPRECATED$ void $add_name$(const std::string& value); + $DEPRECATED$ void $add_name$(std::string&& value); + $DEPRECATED$ void $add_name$(const char* value); + $DEPRECATED$ void $add_name$(absl::string_view value); + $DEPRECATED$ const $pb$::RepeatedPtrField& $name$() const; + $DEPRECATED$ $pb$::RepeatedPtrField* $mutable_name$(); + + private: + const $pb$::RepeatedPtrField& _internal_$name$() const; + $pb$::RepeatedPtrField* _internal_mutable_$name$(); + + public: + )cc"); +} + +void RepeatedStringView::GenerateInlineAccessorDefinitions( + io::Printer* p) const { + p->Emit({{"Get", opts_->safe_boundary_check ? "InternalCheckedGet" : "Get"}, + {"GetExtraArg", + [&] { + p->Emit(opts_->safe_boundary_check + ? ", $pbi$::GetEmptyStringAlreadyInited()" + : ""); + }}}, + R"cc( + inline absl::string_view $Msg$::$name$(int index) const + ABSL_ATTRIBUTE_LIFETIME_BOUND { + $WeakDescriptorSelfPin$; + $annotate_get$; + // @@protoc_insertion_point(field_get:$pkg.Msg.field$) + return _internal_$name_internal$().$Get$(index$GetExtraArg$); + } + inline void $Msg$::set_$name$(int index, const std::string& value) { + $WeakDescriptorSelfPin$; + _internal_mutable_$name_internal$()->Mutable(index)->assign(value); + $annotate_set$; + // @@protoc_insertion_point(field_set:$pkg.Msg.field$) + } + inline void $Msg$::set_$name$(int index, std::string&& value) { + $WeakDescriptorSelfPin$; + _internal_mutable_$name_internal$()->Mutable(index)->assign(std::move(value)); + $annotate_set$; + // @@protoc_insertion_point(field_set:$pkg.Msg.field$) + } + inline void $Msg$::set_$name$(int index, const char* value) { + $WeakDescriptorSelfPin$; + $DCHK$(value != nullptr); + _internal_mutable_$name_internal$()->Mutable(index)->assign(value); + $annotate_set$; + // @@protoc_insertion_point(field_set_char:$pkg.Msg.field$) + } + inline void $Msg$::set_$name$(int index, absl::string_view value) { + $WeakDescriptorSelfPin$; + _internal_mutable_$name_internal$()->Mutable(index)->assign( + value.data(), value.size()); + $annotate_set$; + // @@protoc_insertion_point(field_set_string_piece:$pkg.Msg.field$) + } + inline void $Msg$::add_$name$(const std::string& value) { + $WeakDescriptorSelfPin$; + $TsanDetectConcurrentMutation$; + _internal_mutable_$name_internal$()->Add()->assign(value); + $annotate_add$; + // @@protoc_insertion_point(field_add:$pkg.Msg.field$) + } + inline void $Msg$::add_$name$(std::string&& value) { + $WeakDescriptorSelfPin$; + $TsanDetectConcurrentMutation$; + _internal_mutable_$name_internal$()->Add(std::move(value)); + $annotate_add$; + // @@protoc_insertion_point(field_add:$pkg.Msg.field$) + } + inline void $Msg$::add_$name$(const char* value) { + $WeakDescriptorSelfPin$; + $DCHK$(value != nullptr); + $TsanDetectConcurrentMutation$; + _internal_mutable_$name_internal$()->Add()->assign(value); + $annotate_add$; + // @@protoc_insertion_point(field_add_char:$pkg.Msg.field$) + } + inline void $Msg$::add_$name$(absl::string_view value) { + $WeakDescriptorSelfPin$; + $TsanDetectConcurrentMutation$; + _internal_mutable_$name_internal$()->Add()->assign(value.data(), + value.size()); + $annotate_add$; + // @@protoc_insertion_point(field_add_string_piece:$pkg.Msg.field$) + } + inline const ::$proto_ns$::RepeatedPtrField& + $Msg$::$name$() const ABSL_ATTRIBUTE_LIFETIME_BOUND { + $WeakDescriptorSelfPin$; + $annotate_list$; + // @@protoc_insertion_point(field_list:$pkg.Msg.field$) + return _internal_$name_internal$(); + } + inline ::$proto_ns$::RepeatedPtrField* + $Msg$::mutable_$name$() ABSL_ATTRIBUTE_LIFETIME_BOUND { + $WeakDescriptorSelfPin$; + $annotate_mutable_list$; + // @@protoc_insertion_point(field_mutable_list:$pkg.Msg.field$) + $TsanDetectConcurrentMutation$; + return _internal_mutable_$name_internal$(); + } + )cc"); + if (should_split()) { + p->Emit(R"cc( + inline const $pb$::RepeatedPtrField& + $Msg$::_internal_$name_internal$() const { + $TsanDetectConcurrentRead$; + return *$field_$; + } + inline $pb$::RepeatedPtrField* + $Msg$::_internal_mutable_$name_internal$() { + $TsanDetectConcurrentRead$; + $PrepareSplitMessageForWrite$; + if ($field_$.IsDefault()) { + $field_$.Set( + $pb$::Arena::CreateMessage<$pb$::RepeatedPtrField>( + GetArena())); + } + return $field_$.Get(); + } + )cc"); + } else { + p->Emit(R"cc( + inline const ::$proto_ns$::RepeatedPtrField& + $Msg$::_internal_$name_internal$() const { + $TsanDetectConcurrentRead$; + return $field_$; + } + inline ::$proto_ns$::RepeatedPtrField* + $Msg$::_internal_mutable_$name_internal$() { + $TsanDetectConcurrentRead$; + return &$field_$; + } + )cc"); + } +} + +void RepeatedStringView::GenerateSerializeWithCachedSizesToArray( + io::Printer* p) const { + p->Emit({{"utf8_check", + [&] { + GenerateUtf8CheckCodeForString( + p, field_, options_, false, + "s.data(), static_cast(s.length()),"); + }}}, + R"cc( + for (int i = 0, n = this->_internal_$name$_size(); i < n; ++i) { + const auto& s = this->_internal_$name$().Get(i); + $utf8_check$; + target = stream->Write$DeclaredType$($number$, s, target); + } + )cc"); +} +} // namespace + +std::unique_ptr MakeSingularStringViewGenerator( + const FieldDescriptor* desc, const Options& options, + MessageSCCAnalyzer* scc) { + return absl::make_unique(desc, options, scc); +} + +std::unique_ptr MakeRepeatedStringViewGenerator( + const FieldDescriptor* desc, const Options& options, + MessageSCCAnalyzer* scc) { + return absl::make_unique(desc, options, scc); +} + +} // namespace cpp +} // namespace compiler +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/compiler/cpp/generator.h b/src/google/protobuf/compiler/cpp/generator.h index 05d6c72462..d1a274ae6a 100644 --- a/src/google/protobuf/compiler/cpp/generator.h +++ b/src/google/protobuf/compiler/cpp/generator.h @@ -80,6 +80,9 @@ class PROTOC_EXPORT CppGenerator : public CodeGenerator { return {GetExtensionReflection(pb::cpp)}; } + using CodeGenerator::GetEdition; + using CodeGenerator::GetResolvedSourceFeatures; + private: bool opensource_runtime_ = PROTO2_IS_OSS; std::string runtime_include_base_; diff --git a/src/google/protobuf/sample_messages_edition.proto b/src/google/protobuf/sample_messages_edition.proto index f2d279830d..c0310dac39 100644 --- a/src/google/protobuf/sample_messages_edition.proto +++ b/src/google/protobuf/sample_messages_edition.proto @@ -14,7 +14,10 @@ edition = "2023"; package protobuf_test_messages.edition; +import "google/protobuf/cpp_features.proto"; + option optimize_for = SPEED; +option features.(pb.cpp).string_type = VIEW; // This proto includes every type of field in both singular and repeated // forms. diff --git a/src/google/protobuf/string_view_test.cc b/src/google/protobuf/string_view_test.cc new file mode 100644 index 0000000000..81f781fe61 --- /dev/null +++ b/src/google/protobuf/string_view_test.cc @@ -0,0 +1,241 @@ +#include +#include +#include + +#include +#include +// clang-format off +#include "absl/strings/string_view.h" +// clang-format on +#include "google/protobuf/text_format.h" +#include "google/protobuf/unittest_string_view.pb.h" + +namespace google { +namespace protobuf { +namespace { + +using ::protobuf_unittest::TestStringView; +using ::testing::ElementsAre; +using ::testing::StrEq; + +TEST(StringViewFieldTest, SingularViewGetter) { + TestStringView message; + + ASSERT_TRUE(TextFormat::ParseFromString( + R"pb( + singular_string: "0123456789" + singular_bytes: "012345678901234567890123456789" + )pb", + &message)); + + // singular_string + EXPECT_TRUE(message.has_singular_string()); + + auto singular_string = message.singular_string(); + static_assert( + std::is_same::value, + "unexpected type"); + EXPECT_THAT(singular_string, StrEq("0123456789")); + + // singular_bytes + EXPECT_TRUE(message.has_singular_bytes()); + + auto singular_bytes = message.singular_bytes(); + static_assert( + std::is_same::value, + "unexpected type"); + EXPECT_THAT(singular_bytes, StrEq("012345678901234567890123456789")); +} + +template +void VerifySingularStringSet(TestStringView& message, T&& value, + absl::string_view expected) { + message.set_singular_string(static_cast(value)); + + EXPECT_TRUE(message.has_singular_string()); + EXPECT_THAT(message.singular_string(), StrEq(expected)); + + message.clear_singular_string(); + + EXPECT_FALSE(message.has_singular_string()); + EXPECT_EQ(message.singular_string().size(), 0); +} + +#define STRING_PAYLOAD "012345678901234567890123456789" + +TEST(StringViewFieldTest, SingularSetByStringView) { + TestStringView message; + + absl::string_view value = {STRING_PAYLOAD}; + + VerifySingularStringSet(message, value, value); +} + +TEST(StringViewFieldTest, SingularSetByCharPtr) { + TestStringView message; + + absl::string_view expected = {STRING_PAYLOAD}; + const char* ptr = STRING_PAYLOAD; + + VerifySingularStringSet(message, ptr, expected); +} + +TEST(StringViewFieldTest, SingularSetByConstStringRef) { + TestStringView message; + + std::string value = STRING_PAYLOAD; + const std::string& ref = value; + + VerifySingularStringSet(message, ref, STRING_PAYLOAD); +} + +TEST(StringViewFieldTest, SingularSetByStringMove) { + TestStringView message; + + std::string value = STRING_PAYLOAD; + + VerifySingularStringSet(message, std::move(value), STRING_PAYLOAD); +} + +TEST(StringViewFieldTest, RepeatedViewGetter) { + TestStringView message; + + ASSERT_TRUE(TextFormat::ParseFromString(R"pb( + repeated_string: "foo" + repeated_string: "bar" + repeated_string: "baz" + + repeated_bytes: "000" + repeated_bytes: "111" + repeated_bytes: "222" + repeated_bytes: "333" + repeated_bytes: "444" + )pb", + &message)); + + EXPECT_EQ(message.repeated_string_size(), 3); + + auto repeated_string_0 = message.repeated_string(0); + static_assert( + std::is_same::value, + "unexpected type"); + EXPECT_THAT(repeated_string_0, StrEq("foo")); + EXPECT_THAT(message.repeated_string(), ElementsAre("foo", "bar", "baz")); + + EXPECT_EQ(message.repeated_bytes_size(), 5); + + auto repeated_bytes_2 = message.repeated_bytes(2); + static_assert( + std::is_same::value, + "unexpected type"); + EXPECT_THAT(repeated_bytes_2, StrEq("222")); + EXPECT_THAT(message.repeated_bytes(), + ElementsAre("000", "111", "222", "333", "444")); +} + +TEST(StringViewFieldTest, RepeatedSetByCharPtr) { + TestStringView message; + + const char* ptr0 = "foo"; + const char* ptr1 = "baz"; + const char* ptr2 = STRING_PAYLOAD; + message.add_repeated_string(ptr0); + message.add_repeated_string(ptr1); + message.add_repeated_string(ptr2); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("foo", "baz", STRING_PAYLOAD)); + + message.set_repeated_string(0, ptr1); + message.set_repeated_string(1, ptr2); + message.set_repeated_string(2, ptr0); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("baz", STRING_PAYLOAD, "foo")); +} + +TEST(StringViewFieldTest, RepeatedSetByStringView) { + TestStringView message; + + absl::string_view view0 = "foo"; + absl::string_view view1 = "baz"; + absl::string_view view2 = STRING_PAYLOAD; + message.add_repeated_string(view0); + message.add_repeated_string(view1); + message.add_repeated_string(view2); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("foo", "baz", STRING_PAYLOAD)); + + message.set_repeated_string(0, view1); + message.set_repeated_string(1, view2); + message.set_repeated_string(2, view0); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("baz", STRING_PAYLOAD, "foo")); +} + +TEST(StringViewFieldTest, RepeatedSetByConstStringRef) { + TestStringView message; + + std::string str0 = "foo"; + std::string str1 = "baz"; + std::string str2 = STRING_PAYLOAD; + message.add_repeated_string(str0); + message.add_repeated_string(str1); + message.add_repeated_string(str2); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("foo", "baz", STRING_PAYLOAD)); + + message.set_repeated_string(0, str1); + message.set_repeated_string(1, str2); + message.set_repeated_string(2, str0); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("baz", STRING_PAYLOAD, "foo")); +} + +TEST(StringViewFieldTest, RepeatedSetByStringMove) { + TestStringView message; + + message.add_repeated_string(std::string{"foo"}); + message.add_repeated_string(std::string{"baz"}); + message.add_repeated_string(std::string{STRING_PAYLOAD}); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("foo", "baz", STRING_PAYLOAD)); + + message.set_repeated_string(0, std::string{"baz"}); + message.set_repeated_string(1, std::string{STRING_PAYLOAD}); + message.set_repeated_string(2, std::string{"foo"}); + + EXPECT_THAT(message.repeated_string(), + ElementsAre("baz", STRING_PAYLOAD, "foo")); +} + +TEST(StringViewFieldTest, RepeatedViewSetter) { + TestStringView message; + + message.add_repeated_string("000"); + message.add_repeated_string("111"); + message.add_repeated_string("222"); + + EXPECT_EQ(message.repeated_string_size(), 3); + EXPECT_THAT(message.repeated_string(), ElementsAre("000", "111", "222")); + + for (auto& it : *message.mutable_repeated_string()) { + it.append(it); + } + + EXPECT_EQ(message.repeated_string_size(), 3); + EXPECT_THAT(message.repeated_string(), + ElementsAre("000000", "111111", "222222")); + + message.clear_repeated_string(); + EXPECT_EQ(message.repeated_string_size(), 0); +} + +} // namespace +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/unittest_string_view.proto b/src/google/protobuf/unittest_string_view.proto new file mode 100644 index 0000000000..543ef7c2b7 --- /dev/null +++ b/src/google/protobuf/unittest_string_view.proto @@ -0,0 +1,18 @@ +edition = "2023"; + +package protobuf_unittest; + +import "google/protobuf/cpp_features.proto"; + +option java_multiple_files = true; +option optimize_for = SPEED; +option features.(pb.cpp).string_type = VIEW; + +// NEXT_TAG = 5; +message TestStringView { + string singular_string = 1; + bytes singular_bytes = 2; + + repeated string repeated_string = 3; + repeated bytes repeated_bytes = 4; +}