Automated rollback of commit 2221a0111d.

PiperOrigin-RevId: 495991661
pull/11338/head
Protobuf Team Bot 2 years ago committed by Copybara-Service
parent 883ec1c3ef
commit a99e7088c7
  1. 1
      src/google/protobuf/BUILD.bazel
  2. 26
      src/google/protobuf/compiler/cpp/parse_function_generator.cc
  3. 6
      src/google/protobuf/extension_set.h
  4. 23
      src/google/protobuf/generated_message_reflection.cc
  5. 3
      src/google/protobuf/generated_message_tctable_decl.h
  6. 34
      src/google/protobuf/generated_message_tctable_gen.cc
  7. 8
      src/google/protobuf/generated_message_tctable_gen.h
  8. 12
      src/google/protobuf/generated_message_tctable_impl.h
  9. 31
      src/google/protobuf/generated_message_tctable_lite.cc
  10. 150
      src/google/protobuf/lazy_field_lite.cc
  11. 2
      src/google/protobuf/unittest.proto

@ -332,7 +332,6 @@ cc_library(
":arena_config",
"//src/google/protobuf/io",
"//src/google/protobuf/stubs:lite",
"@com_google_absl//absl/cleanup",
"@com_google_absl//absl/container:flat_hash_set",
"@com_google_absl//absl/numeric:bits",
"@com_google_absl//absl/strings:internal",

@ -92,14 +92,7 @@ class ParseFunctionGenerator::GeneratedOptionProvider final
explicit GeneratedOptionProvider(ParseFunctionGenerator* gen) : gen_(gen) {}
TailCallTableInfo::PerFieldOptions GetForField(
const FieldDescriptor* field) const final {
const auto verify_flag = [&] {
if (IsEagerlyVerifiedLazy(field, gen_->options_, gen_->scc_analyzer_))
return internal::field_layout::kTvEager;
if (IsLazilyVerifiedLazy(field, gen_->options_))
return internal::field_layout::kTvLazy;
return internal::field_layout::TransformValidation{};
};
return {verify_flag(),
return {IsLazy(field, gen_->options_, gen_->scc_analyzer_),
IsStringInlined(field, gen_->options_),
IsImplicitWeakField(field, gen_->options_, gen_->scc_analyzer_),
UseDirectTcParserTable(field, gen_->options_),
@ -631,15 +624,6 @@ void ParseFunctionGenerator::GenerateTailCallTable(Formatter& format) {
QualifiedDefaultInstancePtr(
aux_entry.field->message_type(), options_));
break;
case TailCallTableInfo::kMessageVerifyFunc:
if (aux_entry.field != nullptr) {
format("{$1$::InternalVerify},\n",
QualifiedClassName(aux_entry.field->message_type(),
options_));
} else {
format("{},\n");
}
break;
case TailCallTableInfo::kEnumRange:
format("{$1$, $2$},\n", aux_entry.enum_range.start,
aux_entry.enum_range.size);
@ -773,15 +757,13 @@ static void FormatFieldKind(Formatter& format,
format(" | ::_fl::kRep$1$", rep);
}
static constexpr const char* kXFormNames[2][4] = {
{nullptr, "Default", "Table", "WeakPtr"}, {nullptr, "Eager", "Lazy"}};
static constexpr const char* kXFormNames[] = {nullptr, "Default", "Table",
"WeakPtr"};
static_assert((fl::kTvDefault >> fl::kTvShift) == 1, "");
static_assert((fl::kTvTable >> fl::kTvShift) == 2, "");
static_assert((fl::kTvWeakPtr >> fl::kTvShift) == 3, "");
static_assert((fl::kTvEager >> fl::kTvShift) == 1, "");
static_assert((fl::kTvLazy >> fl::kTvShift) == 2, "");
if (auto* xform = kXFormNames[rep_index == 2][tv_index]) {
if (auto* xform = kXFormNames[tv_index]) {
format(" | ::_fl::kTv$1$", xform);
}
break;

@ -570,9 +570,9 @@ class PROTOBUF_EXPORT ExtensionSet {
virtual void MergeFromMessage(const MessageLite& msg, Arena* arena) = 0;
virtual void Clear() = 0;
virtual const char* _InternalParse(const MessageLite& prototype,
Arena* arena, LazyVerifyOption option,
const char* ptr, ParseContext* ctx) = 0;
virtual const char* _InternalParse(const Message& prototype, Arena* arena,
LazyVerifyOption option, const char* ptr,
ParseContext* ctx) = 0;
virtual uint8_t* WriteMessageToArray(
const MessageLite* prototype, int number, uint8_t* target,
io::EpsCopyOutputStream* stream) const = 0;

@ -3129,11 +3129,8 @@ void Reflection::PopulateTcParseFastEntries(
*fast_entries++ = {GetFastParseFunction(fast_field.func_name),
{fast_field.coded_tag, fast_field.nonfield_info}};
}
} else if (absl::StrContains(fast_field.func_name, "TcParser::FastMl")) {
// We can't use fast parsing for these entries because we can't specify
// the validator.
*fast_entries++ = {internal::TcParser::MiniParse, {}};
} else if (absl::StrContains(fast_field.func_name, "TcParser::FastEv")) {
} else if (fast_field.func_name.find("TcParser::FastEv") !=
fast_field.func_name.npos) {
// We can't use fast parsing for these entries because we can't specify
// the validator. Use the reflection based parser called from MiniParse.
// TODO(b/239592582): Implement a fast parser for these enums.
@ -3171,12 +3168,6 @@ void Reflection::PopulateTcParseEntries(
// Weak fields are handled by the generated fallback function.
// (These are handled by legacy Google-internal logic.)
*entries = {};
} else if (IsLazyField(field)) {
// Lazy fields require validators, which we can't access from reflection.
// We can just handle them in the reflection fallback for now.
*entries = {};
table_info.aux_entries[entry.aux_idx] =
table_info.aux_entries[entry.aux_idx + 1] = {};
} else if (field->type() == field->TYPE_ENUM &&
table_info.aux_entries[entry.aux_idx].type ==
internal::TailCallTableInfo::kEnumValidator) {
@ -3222,7 +3213,6 @@ void Reflection::PopulateTcParseFieldAux(
break;
case internal::TailCallTableInfo::kSubTable:
case internal::TailCallTableInfo::kSubMessageWeak:
case internal::TailCallTableInfo::kMessageVerifyFunc:
GOOGLE_ABSL_LOG(FATAL) << "Not supported";
break;
case internal::TailCallTableInfo::kSubMessage:
@ -3279,14 +3269,7 @@ const internal::TcParseTableBase* Reflection::CreateTcParseTable() const {
explicit ReflectionOptionProvider(const Reflection& ref) : ref_(ref) {}
internal::TailCallTableInfo::PerFieldOptions GetForField(
const FieldDescriptor* field) const final {
const auto verify_flag = [&] {
if (ref_.IsEagerlyVerifiedLazyField(field))
return internal::field_layout::kTvEager;
if (ref_.IsLazilyVerifiedLazyField(field))
return internal::field_layout::kTvLazy;
return internal::field_layout::TransformValidation{};
};
return {verify_flag(),
return {ref_.IsLazyField(field), //
ref_.IsInlined(field), //
// Only LITE can be implicitly weak.

@ -275,8 +275,6 @@ struct alignas(uint64_t) TcParseTableBase {
constexpr FieldAux(FieldAuxDefaultMessage, const void* msg)
: message_default_p(msg) {}
constexpr FieldAux(const TcParseTableBase* table) : table(table) {}
constexpr FieldAux(LazyEagerVerifyFnType verify_func)
: verify_func(verify_func) {}
bool (*enum_validator)(int);
struct {
int16_t start; // minimum enum number (if it fits)
@ -285,7 +283,6 @@ struct alignas(uint64_t) TcParseTableBase {
uint32_t offset;
const void* message_default_p;
const TcParseTableBase* table;
LazyEagerVerifyFnType verify_func;
const MessageLite* message_default() const {
return static_cast<const MessageLite*>(message_default_p);

@ -155,9 +155,7 @@ void PopulateFastFieldEntry(const TailCallTableInfo::FieldEntryInfo& entry,
}
if (field->type() == field->TYPE_MESSAGE ||
field->type() == field->TYPE_GROUP) {
name.append(options.lazy_opt != 0 ? "l"
: options.use_direct_tcparser_table ? "t"
: "d");
name.append(options.use_direct_tcparser_table ? "t" : "d");
}
// The field implementation functions are prefixed by cardinality:
@ -184,7 +182,7 @@ bool IsFieldEligibleForFastParsing(
// Map, oneof, weak, and lazy fields are not handled on the fast path.
if (field->is_map() || field->real_containing_oneof() ||
field->options().weak() || options.is_implicitly_weak ||
options.should_split) {
options.is_lazy || options.should_split) {
return false;
}
@ -332,10 +330,13 @@ std::vector<TailCallTableInfo::FastFieldInfo> SplitFastFieldsForSize(
// Filter out fields that will be handled by mini parsing.
std::vector<const FieldDescriptor*> FilterMiniParsedFields(
const std::vector<const FieldDescriptor*>& fields,
const TailCallTableInfo::OptionProvider& option_provider) {
const TailCallTableInfo::OptionProvider& option_provider
) {
std::vector<const FieldDescriptor*> generated_fallback_fields;
for (const auto* field : fields) {
auto options = option_provider.GetForField(field);
bool handled = false;
switch (field->type()) {
case FieldDescriptor::TYPE_DOUBLE:
@ -363,7 +364,7 @@ std::vector<const FieldDescriptor*> FilterMiniParsedFields(
case FieldDescriptor::TYPE_MESSAGE:
case FieldDescriptor::TYPE_GROUP:
// TODO(b/210762816): support remaining field types.
if (field->is_map() || field->options().weak()) {
if (field->is_map() || field->options().weak() || options.is_lazy) {
handled = false;
} else {
handled = true;
@ -643,11 +644,10 @@ uint16_t MakeTypeCardForField(
type_card |= fl::kMap;
} else {
type_card |= fl::kMessage;
if (options.lazy_opt != 0) {
GOOGLE_ABSL_CHECK(options.lazy_opt == field_layout::kTvEager ||
options.lazy_opt == field_layout::kTvLazy);
type_card |= +fl::kRepLazy | options.lazy_opt;
} else {
if (options.is_lazy) {
type_card |= fl::kRepLazy;
}
if (options.is_implicitly_weak) {
type_card |= fl::kTvWeakPtr;
} else if (options.use_direct_tcparser_table) {
@ -656,7 +656,6 @@ uint16_t MakeTypeCardForField(
type_card |= fl::kTvDefault;
}
}
}
break;
}
@ -723,12 +722,8 @@ TailCallTableInfo::TailCallTableInfo(
} else if (field->options().weak()) {
// Don't generate anything for weak fields. They are handled by the
// generated fallback.
} else if (options.lazy_opt != 0) {
field_entries.back().aux_idx = aux_entries.size();
aux_entries.push_back({kSubMessage, {field}});
aux_entries.push_back(
{kMessageVerifyFunc,
{options.lazy_opt == field_layout::kTvEager ? field : nullptr}});
} else if (options.is_lazy) {
// Lazy fields are handled by the generated fallback function.
} else {
field_entries.back().aux_idx = aux_entries.size();
aux_entries.push_back({options.is_implicitly_weak ? kSubMessageWeak
@ -819,7 +814,8 @@ TailCallTableInfo::TailCallTableInfo(
// Filter out fields that are handled by MiniParse. We don't need to generate
// a fallback for these, which saves code size.
fallback_fields = FilterMiniParsedFields(ordered_fields, option_provider);
fallback_fields = FilterMiniParsedFields(ordered_fields, option_provider
);
num_to_entry_table = MakeNumToEntryTable(ordered_fields);
GOOGLE_ABSL_CHECK_EQ(field_entries.size(), ordered_fields.size());

@ -50,15 +50,10 @@ namespace google {
namespace protobuf {
namespace internal {
namespace field_layout {
enum TransformValidation : uint16_t;
} // namespace field_layout
// Helper class for generating tailcall parsing functions.
struct PROTOBUF_EXPORT TailCallTableInfo {
struct PerFieldOptions {
// kTvEager, kTvLazy, or 0
field_layout::TransformValidation lazy_opt;
bool is_lazy;
bool is_string_inlined;
bool is_implicitly_weak;
bool use_direct_tcparser_table;
@ -108,7 +103,6 @@ struct PROTOBUF_EXPORT TailCallTableInfo {
kSubMessage,
kSubTable,
kSubMessageWeak,
kMessageVerifyFunc,
kEnumRange,
kEnumValidator,
kNumericOffset,

@ -170,10 +170,6 @@ enum TransformValidation : uint16_t {
kTvDefault = 1 << kTvShift, // Aux has default_instance*
kTvTable = 2 << kTvShift, // Aux has TcParseTableBase*
kTvWeakPtr = 3 << kTvShift, // Aux has default_instance** (for weak)
// Lazy message fields:
kTvEager = 1 << kTvShift,
kTvLazy = 2 << kTvShift,
};
static_assert((kTvEnum & kTvRange) != 0,
@ -459,7 +455,7 @@ class PROTOBUF_EXPORT TcParser final {
// Functions referenced by generated fast tables (message types):
// M: message G: group
// d: default* t: TcParseTable* (the contents of aux) l: lazy
// d: default* t: TcParseTable* (the contents of aux)
// S: singular R: repeated
// 1/2: tag length (bytes)
static const char* FastMdS1(PROTOBUF_TC_PARAM_DECL);
@ -471,9 +467,6 @@ class PROTOBUF_EXPORT TcParser final {
static const char* FastGtS1(PROTOBUF_TC_PARAM_DECL);
static const char* FastGtS2(PROTOBUF_TC_PARAM_DECL);
static const char* FastMlS1(PROTOBUF_TC_PARAM_DECL);
static const char* FastMlS2(PROTOBUF_TC_PARAM_DECL);
static const char* FastMdR1(PROTOBUF_TC_PARAM_DECL);
static const char* FastMdR2(PROTOBUF_TC_PARAM_DECL);
static const char* FastGdR1(PROTOBUF_TC_PARAM_DECL);
@ -550,8 +543,6 @@ class PROTOBUF_EXPORT TcParser final {
static inline const char* SingularParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType, bool group_coding, bool aux_is_table>
static inline const char* RepeatedParseMessageAuxImpl(PROTOBUF_TC_PARAM_DECL);
template <typename TagType>
static inline const char* LazyMessage(PROTOBUF_TC_PARAM_DECL);
template <typename TagType>
static const char* FastEndGroupImpl(PROTOBUF_TC_PARAM_DECL);
@ -720,7 +711,6 @@ class PROTOBUF_EXPORT TcParser final {
template <bool is_split>
static const char* MpMessage(PROTOBUF_TC_PARAM_DECL);
static const char* MpRepeatedMessage(PROTOBUF_TC_PARAM_DECL);
static const char* MpLazyMessage(PROTOBUF_TC_PARAM_DECL);
static const char* MpFallback(PROTOBUF_TC_PARAM_DECL);
};

@ -34,7 +34,6 @@
#include <type_traits>
#include <utility>
#include "absl/cleanup/cleanup.h"
#include "google/protobuf/generated_message_tctable_decl.h"
#include "google/protobuf/generated_message_tctable_impl.h"
#include "google/protobuf/inlined_string_field.h"
@ -465,20 +464,6 @@ PROTOBUF_NOINLINE const char* TcParser::FastGtS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_TC_PARAM_PASS);
}
template <typename TagType>
const char* TcParser::LazyMessage(PROTOBUF_TC_PARAM_DECL) {
GOOGLE_ABSL_LOG(FATAL) << "Unimplemented";
return nullptr;
}
PROTOBUF_NOINLINE const char* TcParser::FastMlS1(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return LazyMessage<uint8_t>(PROTOBUF_TC_PARAM_PASS);
}
PROTOBUF_NOINLINE const char* TcParser::FastMlS2(PROTOBUF_TC_PARAM_DECL) {
PROTOBUF_MUSTTAIL return LazyMessage<uint16_t>(PROTOBUF_TC_PARAM_PASS);
}
template <typename TagType, bool group_coding, bool aux_is_table>
inline PROTOBUF_ALWAYS_INLINE const char* TcParser::RepeatedParseMessageAuxImpl(
PROTOBUF_TC_PARAM_DECL) {
@ -2301,11 +2286,6 @@ PROTOBUF_NOINLINE const char* TcParser::MpRepeatedString(
return ToParseLoop(PROTOBUF_TC_PARAM_PASS);
}
const char* TcParser::MpLazyMessage(PROTOBUF_TC_PARAM_DECL) {
GOOGLE_ABSL_LOG(FATAL) << "Unimplemented";
return nullptr;
}
template <bool is_split>
PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
const auto& entry = RefAt<FieldEntry>(table, data.entry_offset());
@ -2334,13 +2314,12 @@ PROTOBUF_NOINLINE const char* TcParser::MpMessage(PROTOBUF_TC_PARAM_DECL) {
goto fallback;
}
break;
case field_layout::kRepLazy:
if (decoded_wiretype != WireFormatLite::WIRETYPE_LENGTH_DELIMITED) {
goto fallback;
}
PROTOBUF_MUSTTAIL return MpLazyMessage(PROTOBUF_TC_PARAM_PASS);
default: {
fallback:
PROTOBUF_MUSTTAIL return MpFallback(PROTOBUF_TC_PARAM_PASS);
// Lazy and implicit weak fields are handled by generated code:
// TODO(b/210762816): support these.
PROTOBUF_MUSTTAIL return table->fallback(PROTOBUF_TC_PARAM_PASS);
}
}
const bool is_oneof = card == field_layout::kFcOneof;

@ -1,150 +0,0 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2022 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#include <limits>
#include <memory>
#include <utility>
#include "google/protobuf/io/coded_stream.h"
#include "google/protobuf/io/zero_copy_stream_impl_lite.h"
#include "google/protobuf/lazy_field.h"
// Must be included last.
// clang-format off
#include "google/protobuf/port_def.inc"
// clang-format on
namespace google {
namespace protobuf {
namespace internal {
namespace {
// Returns the initial depth for a local context.
// --The default depth for lazily verified LazyField.
// --Eagerly verified LazyField has no limit (max int).
inline int GetInitDepth(LazyVerifyOption option) {
return option == LazyVerifyOption::kLazy
? io::CodedInputStream::GetDefaultRecursionLimit()
: std::numeric_limits<int>::max();
} // namespace
template <typename T>
inline bool ParseWithOuterContextImpl(const T& input, LazyVerifyOption option,
ParseContext* ctx, MessageLite* message) {
GOOGLE_ABSL_DCHECK(ctx == nullptr || !ctx->AliasingEnabled());
// Create local context with depth.
const char* ptr;
ParseContext local_ctx =
ctx != nullptr ? ctx->Spawn(&ptr, input)
: ParseContext(GetInitDepth(option), false, &ptr, input);
if (ctx == nullptr ||
ctx->lazy_parse_mode() == ParseContext::LazyParseMode::kEagerVerify) {
// Unparsed data is already verified at parsing. Disable eager-verification.
(void)local_ctx.set_lazy_parse_mode(ParseContext::LazyParseMode::kLazy);
}
ptr = message->_InternalParse(ptr, &local_ctx);
if (ctx != nullptr && local_ctx.missing_required_fields()) {
ctx->SetMissingRequiredFields();
}
return ptr != nullptr &&
(local_ctx.EndedAtEndOfStream() || local_ctx.EndedAtLimit());
}
} // namespace
namespace {
class ByPrototype {
public:
explicit ByPrototype(const MessageLite* prototype) : prototype_(prototype) {}
MessageLite* New(Arena* arena) const { return prototype_->New(arena); }
const MessageLite& Default() const { return *prototype_; }
private:
const MessageLite* prototype_;
};
} // namespace
const MessageLite& LazyField::GetByPrototype(const MessageLite& prototype,
Arena* arena,
LazyVerifyOption option,
ParseContext* ctx) const {
return GetGeneric(ByPrototype(&prototype), arena, option, ctx);
}
MessageLite* LazyField::MutableByPrototype(const MessageLite& prototype,
Arena* arena,
LazyVerifyOption option,
ParseContext* ctx) {
return MutableGeneric(ByPrototype(&prototype), arena, option, ctx);
}
MessageLite* LazyField::ReleaseByPrototype(const MessageLite& prototype,
Arena* arena,
LazyVerifyOption option) {
return ReleaseGeneric(ByPrototype(&prototype), arena, option);
}
MessageLite* LazyField::UnsafeArenaReleaseByPrototype(
const MessageLite& prototype, Arena* arena, LazyVerifyOption option) {
return UnsafeArenaReleaseGeneric(ByPrototype(&prototype), arena, option);
}
bool LazyField::ParseWithOuterContext(MessageLite* message,
LazyVerifyOption option,
ParseContext* ctx) const {
absl::optional<absl::string_view> flat = unparsed_.TryFlat();
if (flat.has_value()) {
return ParseWithOuterContextImpl(*flat, option, ctx, message);
}
io::CordInputStream input(unparsed_);
return ParseWithOuterContextImpl(&input, option, ctx, message);
}
void LazyField::LogParseError(const MessageLite* message) {
GOOGLE_ABSL_LOG_EVERY_N(INFO, 100)
<< "Lazy parsing failed for " << message->GetTypeName()
<< " error=" << message->InitializationErrorString()
<< " (N = " << COUNTER << ")";
}
} // namespace internal
} // namespace protobuf
} // namespace google
#include "google/protobuf/port_undef.inc"

@ -581,11 +581,9 @@ message TestDupFieldNumber { // NO_PROTO1
// Additional messages for testing lazy fields.
message TestEagerMessage {
optional TestAllTypes sub_message = 1 [lazy=false];
optional TestAllTypes sub_message_slow = 100000 [lazy=false];
}
message TestLazyMessage {
optional TestAllTypes sub_message = 1 [lazy=true];
optional TestAllTypes sub_message_slow = 100000 [lazy=true];
}
message TestEagerMaybeLazy {
message NestedMessage {

Loading…
Cancel
Save