From 908de8d74aedf92bf1dc28713678ab6d81207558 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Fri, 21 Apr 2023 14:54:12 -0700 Subject: [PATCH] Automated rollback of commit 1e67de4ed5a52b52b0fd0aba22db531ac0edfb8e. PiperOrigin-RevId: 526143586 --- src/google/protobuf/generated_enum_util.cc | 131 ----------------- src/google/protobuf/generated_enum_util.h | 58 -------- .../protobuf/generated_enum_util_test.cc | 134 ------------------ 3 files changed, 323 deletions(-) delete mode 100644 src/google/protobuf/generated_enum_util_test.cc diff --git a/src/google/protobuf/generated_enum_util.cc b/src/google/protobuf/generated_enum_util.cc index 65ce16dd28..ab21577fff 100644 --- a/src/google/protobuf/generated_enum_util.cc +++ b/src/google/protobuf/generated_enum_util.cc @@ -31,16 +31,9 @@ #include "google/protobuf/generated_enum_util.h" #include -#include -#include -#include "absl/log/absl_check.h" -#include "absl/types/optional.h" #include "google/protobuf/generated_message_util.h" -// Must be included last. -#include "google/protobuf/port_def.inc" - namespace google { namespace protobuf { namespace internal { @@ -97,130 +90,6 @@ bool InitializeEnumStrings( return true; } -bool ValidateEnum(int value, const uint16_t* data) { - return ValidateEnumInlined(value, data); -} - -struct EytzingerLayoutSorter { - absl::Span input; - absl::Span output; - int i; - - void Sort(size_t output_index = 1) { - if (output_index <= input.size()) { - Sort(2 * output_index); - output[output_index * 2 - 2] = input[i]; - output[output_index * 2 - 1] = input[i] >> 16; - i++; - Sort(2 * output_index + 1); - } - } -}; - -std::vector GenerateEnumData(const std::vector& values) { - const auto sorted_and_unique = [&] { - for (size_t i = 0; i + 1 < values.size(); ++i) { - if (values[i] >= values[i + 1]) return false; - } - return true; - }; - ABSL_DCHECK(sorted_and_unique()); - std::vector fallback_values_too_large, fallback_values_after_bitmap; - std::vector bitmap_values; - absl::optional start_sequence; - uint16_t sequence_length = 0; - for (int32_t v : values) { - if (static_cast(v) != v) { - fallback_values_too_large.push_back(v); - continue; - } - // If we don't yet have a sequence, start it. - if (!start_sequence.has_value()) { - start_sequence = v; - sequence_length = 1; - continue; - } - // If we can extend the sequence, do so. - if (v == *start_sequence + sequence_length && sequence_length < 0xFFFF) { - ++sequence_length; - continue; - } - - // We adjust the bitmap values to be relative to the end of the sequence. - const auto adjust = [&](int32_t v) -> uint32_t { - // Cast to int64_t first to avoid overflow. The result is guaranteed to be - // positive and fit in uint32_t. - return static_cast(v) - *start_sequence - sequence_length; - }; - const uint32_t adjusted = adjust(v); - - const auto add_bit = [&](uint32_t bit) { - bitmap_values[bit / 16] |= 1 << (bit % 16); - }; - - // If we can fit it on the already allocated bitmap, do so. - if (adjusted < 16 * bitmap_values.size()) { - // We can fit it in the existing bitmap. - ABSL_DCHECK_EQ(fallback_values_after_bitmap.size(), 0); - add_bit(adjusted); - continue; - } - - // We can't fit in the sequence and we can't fit in the current bitmap. - // Evaluate if it is better to add to fallback, or to collapse all the - // fallback values after the bitmap into the bitmap. - const size_t cost_if_fallback = - sizeof(uint16_t) * bitmap_values.size() + - sizeof(int32_t) * (1 + fallback_values_after_bitmap.size()); - const size_t rounded_bitmap_size = (adjusted + 1 + 15) / 16; - const size_t cost_if_collapse = sizeof(uint16_t) * rounded_bitmap_size; - - if (cost_if_collapse <= cost_if_fallback) { - // Collapse the existing values, and add the new one. - ABSL_DCHECK_GT(rounded_bitmap_size, bitmap_values.size()); - bitmap_values.resize(rounded_bitmap_size); - for (int32_t to_collapse : fallback_values_after_bitmap) { - add_bit(adjust(to_collapse)); - } - fallback_values_after_bitmap.clear(); - add_bit(adjusted); - } else { - fallback_values_after_bitmap.push_back(v); - } - } - - std::vector fallback_values; - if (fallback_values_after_bitmap.empty()) { - fallback_values = std::move(fallback_values_too_large); - } else if (fallback_values_too_large.empty()) { - fallback_values = std::move(fallback_values_after_bitmap); - } else { - fallback_values.resize(fallback_values_too_large.size() + - fallback_values_after_bitmap.size()); - std::merge(fallback_values_too_large.begin(), - fallback_values_too_large.end(), - fallback_values_after_bitmap.begin(), - fallback_values_after_bitmap.end(), &fallback_values[0]); - } - - std::vector output( - 4 /* seq start + seq len + bitmap len + ordered len */ + - bitmap_values.size() + 2 * fallback_values.size()); - - output[0] = start_sequence.value_or(0); - output[1] = sequence_length; - output[2] = 16 * bitmap_values.size(); - output[3] = fallback_values.size(); - auto sorted_start = - std::copy(bitmap_values.begin(), bitmap_values.end(), output.data() + 4); - - EytzingerLayoutSorter{ - fallback_values, absl::MakeSpan(sorted_start, 2 * fallback_values.size())} - .Sort(); - - return output; -} - } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/generated_enum_util.h b/src/google/protobuf/generated_enum_util.h index d57ca1e8c6..1f06f2f6a7 100644 --- a/src/google/protobuf/generated_enum_util.h +++ b/src/google/protobuf/generated_enum_util.h @@ -31,10 +31,7 @@ #ifndef GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__ #define GOOGLE_PROTOBUF_GENERATED_ENUM_UTIL_H__ -#include -#include #include -#include #include "absl/strings/string_view.h" #include "google/protobuf/message_lite.h" @@ -78,61 +75,6 @@ PROTOBUF_EXPORT bool InitializeEnumStrings( const EnumEntry* enums, const int* sorted_indices, size_t size, internal::ExplicitlyConstructed* enum_strings); -// The enum validation format is split in 3 parts: -// - A dense sequence, with start+length -// - A variable size presence bitmap. -// - A variable size sorted int32_t set for everything else. The int32's are -// stored in 2 uint16_t each in little endian. -// -// The values are as follows: -// -// 0 - [ sequence start (int16_t) ] -// 1 - [ sequence length (uint16_t) ] -// 2 - [ bitmap length in bits (uint16_t) ] -// 3 - [ ordered length (uint16_t) ] -// x - [ variable length bitmap ] -// y - [ variable length of int32_t values ] -// -// Where the bitmap starts right after the end of the sequence. -PROTOBUF_EXPORT bool ValidateEnum(int value, const uint16_t* data); -PROTOBUF_EXPORT std::vector GenerateEnumData( - const std::vector& values); - -inline PROTOBUF_ALWAYS_INLINE bool ValidateEnumInlined(int value, - const uint16_t* data) { - const int16_t min_seq = static_cast(data[0]); - const uint16_t length_seq = data[1]; - uint64_t adjusted = - static_cast(static_cast(value)) - min_seq; - if (PROTOBUF_PREDICT_TRUE(adjusted < length_seq)) { - return true; - } - - const uint16_t length_bitmap = data[2]; - adjusted -= length_seq; - if (PROTOBUF_PREDICT_TRUE(adjusted < length_bitmap)) { - data += 4; -#if defined(__x86_64__) && defined(__GNUC__) - bool result; - asm("bt %1, %2" : "=@ccc"(result) : "r"(adjusted), "m"(*data)); - return result; -#else - return data[adjusted / 16] & (1 << (adjusted % 16)); -#endif - } - const uint16_t num_sorted = data[3]; - data += 4 + length_bitmap / 16; - size_t pos = 0; - while (pos < num_sorted) { - auto it = data + pos * 2; - auto sample = - (static_cast(it[1]) << 16) | static_cast(it[0]); - if (sample == value) return true; - pos = 2 * pos + (sample > value ? 1 : 2); - } - return false; -} - } // namespace internal } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/generated_enum_util_test.cc b/src/google/protobuf/generated_enum_util_test.cc deleted file mode 100644 index 4949abd447..0000000000 --- a/src/google/protobuf/generated_enum_util_test.cc +++ /dev/null @@ -1,134 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2023 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include "google/protobuf/generated_enum_util.h" - -#include -#include -#include -#include -#include - -#include -#include -#include "absl/container/flat_hash_set.h" -#include "absl/types/span.h" - - -// Must be included last. -#include "google/protobuf/port_def.inc" - -using testing::ElementsAre; -using testing::ElementsAreArray; -using testing::UnorderedElementsAreArray; - -namespace google { -namespace protobuf { -namespace internal { -namespace { - -void ValidValues(absl::Span data, int min, int max, - absl::flat_hash_set& out) { - for (int64_t i = min; i <= max; ++i) { - if (ValidateEnum(i, data.begin())) out.insert(i); - } -} - -std::vector ValidValues(absl::Span data, int min, - int max) { - absl::flat_hash_set s; - ValidValues(data, min, max, s); - std::vector out(s.begin(), s.end()); - std::sort(out.begin(), out.end()); - return out; -} - -TEST(ValidateEnumTest, SequentialRangeTest) { - EXPECT_THAT(ValidValues({0, 0, 0, 0}, -100, 100), ElementsAre()); - EXPECT_THAT(ValidValues({5, 3, 0, 0}, -100, 100), ElementsAre(5, 6, 7)); - EXPECT_THAT(ValidValues({static_cast(-2), 10, 0, 0}, -100, 100), - ElementsAre(-2, -1, 0, 1, 2, 3, 4, 5, 6, 7)); -} - -TEST(ValidateEnumTest, BitmapRangeTest) { - EXPECT_THAT(ValidValues({0, 0, 16, 0, 0b10011010101}, -100, 100), - ElementsAre(0, 2, 4, 6, 7, 10)); - EXPECT_THAT(ValidValues({0, 0, 48, 0, 1 << 4, 1 << 5, 1 << 6}, -100, 100), - ElementsAre(4, 16 + 5, 32 + 6)); -} - -TEST(ValidateEnumTest, GenerateEnumDataSequential) { - EXPECT_THAT(GenerateEnumData({0, 1, 2, 3}), ElementsAre(0, 4, 0, 0)); - EXPECT_THAT(GenerateEnumData({-2, -1, 0, 1, 2, 3}), - ElementsAre(static_cast(-2), 6, 0, 0)); -} - -void TestRoundTrip(const std::vector& values) { - auto encoded = GenerateEnumData(values); - - absl::flat_hash_set s; - // Add a few test values in case `values` is empty. - ValidValues(encoded, -100, 100, s); - - // We look at a few values around the expected ones. - // We could in theory test the whole int32_t domain, but that takes too long - // to run. - for (int32_t v : values) { - int32_t min = - std::max(static_cast(v) - 100, - static_cast(std::numeric_limits::min())); - int32_t max = - std::min(static_cast(v) + 100, - static_cast(std::numeric_limits::max())); - ValidValues(encoded, min, max, s); - } - - EXPECT_THAT(s, UnorderedElementsAreArray(values)) - << testing::PrintToString(encoded); -} - -TEST(ValidateEnumTest, GenerateEnumDataBitmap) { - EXPECT_THAT(GenerateEnumData({0, 1, 2, 4, 8, 16}), - ElementsAre(0, 3, 16, 0, 0b10000000100010)); - TestRoundTrip({}); - TestRoundTrip({0, 1, 2, 4, 8, 16}); - TestRoundTrip({0, 1, 2, 4, 8, 16, 32, 64, 128, 256}); - TestRoundTrip({10000, 10001, 10002, 10004, 10006, 10008, 10010}); - TestRoundTrip({std::numeric_limits::min(), -123123, -123, 213, - 213213, std::numeric_limits::max()}); -} - - -} // namespace -} // namespace internal -} // namespace protobuf -} // namespace google - -#include "google/protobuf/port_undef.inc"