diff --git a/CHANGES.txt b/CHANGES.txt index a7d74a7eee..d8ed3e99bb 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,6 +1,7 @@ 2022-07-01 Unreleased version C++ + * cpp_generated_lib_linked support is removed in protoc * Reduced .pb.o object file size slightly by explicitly instantiating InternalMetadata templates in the runtime. * Add C++20 keywords guarded by PROTOBUF_FUTURE_CPP20_KEYWORDS @@ -11,6 +12,7 @@ * Hide C++ RepeatedField::UnsafeArenaSwap Kotlin + * Suppress deprecation warnings in Kotlin generated code. * Kotlin generated code comments now use kdoc format instead of javadoc. * Escape keywords in package names in proto generated code * Add Kotlin enum int value getters and setters diff --git a/python/google/protobuf/descriptor.py b/python/google/protobuf/descriptor.py index fa7a5b96d5..997f6db6f5 100644 --- a/python/google/protobuf/descriptor.py +++ b/python/google/protobuf/descriptor.py @@ -1021,13 +1021,7 @@ class FileDescriptor(DescriptorBase): # FileDescriptor() is called from various places, not only from generated # files, to register dynamic proto files and messages. # pylint: disable=g-explicit-bool-comparison - if serialized_pb == b'': - # Cpp generated code must be linked in if serialized_pb is '' - try: - return _message.default_pool.FindFileByName(name) - except KeyError: - raise RuntimeError('Please link in cpp generated lib for %s' % (name)) - elif serialized_pb: + if serialized_pb: return _message.default_pool.AddSerializedFile(serialized_pb) else: return super(FileDescriptor, cls).__new__(cls) diff --git a/python/google/protobuf/internal/message_test.py b/python/google/protobuf/internal/message_test.py index 624dbdb152..0a6c0b1ab2 100644 --- a/python/google/protobuf/internal/message_test.py +++ b/python/google/protobuf/internal/message_test.py @@ -1768,6 +1768,19 @@ class Proto3Test(unittest.TestCase): with self.assertRaises(TypeError): 123 in msg.map_string_string + def testScalarMapComparison(self): + msg1 = map_unittest_pb2.TestMap() + msg2 = map_unittest_pb2.TestMap() + + self.assertEqual(msg1.map_int32_int32, msg2.map_int32_int32) + + def testMessageMapComparison(self): + msg1 = map_unittest_pb2.TestMap() + msg2 = map_unittest_pb2.TestMap() + + self.assertEqual(msg1.map_int32_foreign_message, + msg2.map_int32_foreign_message) + def testMapGet(self): # Need to test that get() properly returns the default, even though the dict # has defaultdict-like semantics. diff --git a/src/google/protobuf/arenaz_sampler.cc b/src/google/protobuf/arenaz_sampler.cc index 53b524e1c3..ef675182f9 100644 --- a/src/google/protobuf/arenaz_sampler.cc +++ b/src/google/protobuf/arenaz_sampler.cc @@ -33,6 +33,7 @@ #include #include #include +#include // Must be included last. @@ -74,11 +75,15 @@ PROTOBUF_THREAD_LOCAL SamplingState global_sampling_state = { ThreadSafeArenaStats::ThreadSafeArenaStats() { PrepareForSampling(0); } ThreadSafeArenaStats::~ThreadSafeArenaStats() = default; -void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { +void ThreadSafeArenaStats::BlockStats::PrepareForSampling() { num_allocations.store(0, std::memory_order_relaxed); - bytes_used.store(0, std::memory_order_relaxed); bytes_allocated.store(0, std::memory_order_relaxed); + bytes_used.store(0, std::memory_order_relaxed); bytes_wasted.store(0, std::memory_order_relaxed); +} + +void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { + for (auto& blockstats : block_histogram) blockstats.PrepareForSampling(); max_block_size.store(0, std::memory_order_relaxed); thread_ids.store(0, std::memory_order_relaxed); weight = stride; @@ -88,12 +93,41 @@ void ThreadSafeArenaStats::PrepareForSampling(int64_t stride) { depth = absl::GetStackTrace(stack, kMaxStackDepth, /* skip_count= */ 0); } +size_t ThreadSafeArenaStats::FindBin(size_t bytes) { + if (bytes <= kMaxSizeForBinZero) return 0; + if (bytes <= kMaxSizeForPenultimateBin) { + // absl::bit_width() returns one plus the base-2 logarithm of x, with any + // fractional part discarded. + return absl::bit_width(absl::bit_ceil(bytes)) - kLogMaxSizeForBinZero - 1; + } + return kBlockHistogramBins - 1; +} + +std::pair ThreadSafeArenaStats::MinMaxBlockSizeForBin( + size_t bin) { + ABSL_ASSERT(bin < kBlockHistogramBins); + if (bin == 0) return {1, kMaxSizeForBinZero}; + if (bin < kBlockHistogramBins - 1) { + return {(1 << (kLogMaxSizeForBinZero + bin - 1)) + 1, + 1 << (kLogMaxSizeForBinZero + bin)}; + } + return {kMaxSizeForPenultimateBin + 1, std::numeric_limits::max()}; +} + void RecordAllocateSlow(ThreadSafeArenaStats* info, size_t used, size_t allocated, size_t wasted) { - info->num_allocations.fetch_add(1, std::memory_order_relaxed); - info->bytes_used.fetch_add(used, std::memory_order_relaxed); - info->bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); - info->bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); + // Update the allocated bytes for the current block. + ThreadSafeArenaStats::BlockStats& curr = + info->block_histogram[ThreadSafeArenaStats::FindBin(allocated)]; + curr.bytes_allocated.fetch_add(allocated, std::memory_order_relaxed); + curr.num_allocations.fetch_add(1, std::memory_order_relaxed); + + // Update the used and wasted bytes for the previous block. + ThreadSafeArenaStats::BlockStats& prev = + info->block_histogram[ThreadSafeArenaStats::FindBin(used + wasted)]; + prev.bytes_used.fetch_add(used, std::memory_order_relaxed); + prev.bytes_wasted.fetch_add(wasted, std::memory_order_relaxed); + if (info->max_block_size.load(std::memory_order_relaxed) < allocated) { info->max_block_size.store(allocated, std::memory_order_relaxed); } diff --git a/src/google/protobuf/arenaz_sampler.h b/src/google/protobuf/arenaz_sampler.h index 43d1acb72f..c608bcd77d 100644 --- a/src/google/protobuf/arenaz_sampler.h +++ b/src/google/protobuf/arenaz_sampler.h @@ -31,9 +31,11 @@ #ifndef GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__ #define GOOGLE_PROTOBUF_SRC_GOOGLE_PROTOBUF_ARENAZ_SAMPLER_H__ +#include #include #include #include +#include // Must be included last. @@ -65,10 +67,28 @@ struct ThreadSafeArenaStats // These fields are mutated by the various Record* APIs and need to be // thread-safe. - std::atomic num_allocations; - std::atomic bytes_used; - std::atomic bytes_allocated; - std::atomic bytes_wasted; + struct BlockStats { + std::atomic num_allocations; + std::atomic bytes_allocated; + std::atomic bytes_used; + std::atomic bytes_wasted; + + void PrepareForSampling(); + }; + + // block_histogram is a kBlockHistogramBins sized histogram. The zeroth bin + // stores info about blocks of size \in [1, 1 << kLogMaxSizeForBinZero]. Bin + // i, where i > 0, stores info for blocks of size \in (max_size_bin (i-1), + // 1 << (kLogMaxSizeForBinZero + i)]. The final bin stores info about blocks + // of size \in [kMaxSizeForPenultimateBin + 1, + // std::numeric_limits::max()]. + static constexpr size_t kBlockHistogramBins = 15; + static constexpr size_t kLogMaxSizeForBinZero = 7; + static constexpr size_t kMaxSizeForBinZero = (1 << kLogMaxSizeForBinZero); + static constexpr size_t kMaxSizeForPenultimateBin = + 1 << (kLogMaxSizeForBinZero + kBlockHistogramBins - 2); + std::array block_histogram; + // Records the largest block allocated for the arena. std::atomic max_block_size; // Bit `i` is set to 1 indicates that a thread with `tid % 63 = i` accessed @@ -90,6 +110,13 @@ struct ThreadSafeArenaStats if (PROTOBUF_PREDICT_TRUE(info == nullptr)) return; RecordAllocateSlow(info, used, allocated, wasted); } + + // Returns the bin for the provided size. + static size_t FindBin(size_t bytes); + + // Returns the min and max bytes that can be stored in the histogram for + // blocks in the provided bin. + static std::pair MinMaxBlockSizeForBin(size_t bin); }; struct SamplingState { diff --git a/src/google/protobuf/arenaz_sampler_test.cc b/src/google/protobuf/arenaz_sampler_test.cc index 774e70da6f..67570e87a4 100644 --- a/src/google/protobuf/arenaz_sampler_test.cc +++ b/src/google/protobuf/arenaz_sampler_test.cc @@ -30,6 +30,8 @@ #include +#include +#include #include #include #include @@ -37,7 +39,6 @@ #include #include -#include // Must be included last. @@ -57,10 +58,17 @@ class ThreadSafeArenaStatsHandlePeer { return h->info_; } }; + std::vector GetBytesAllocated(ThreadSafeArenazSampler* s) { std::vector res; s->Iterate([&](const ThreadSafeArenaStats& info) { - res.push_back(info.bytes_allocated.load(std::memory_order_acquire)); + for (const auto& block_stats : info.block_histogram) { + size_t bytes_allocated = + block_stats.bytes_allocated.load(std::memory_order_acquire); + if (bytes_allocated != 0) { + res.push_back(bytes_allocated); + } + } }); return res; } @@ -69,7 +77,8 @@ ThreadSafeArenaStats* Register(ThreadSafeArenazSampler* s, size_t size, int64_t stride) { auto* info = s->Register(stride); assert(info != nullptr); - info->bytes_allocated.store(size); + info->block_histogram[0].bytes_allocated.store(size, + std::memory_order_relaxed); return info; } @@ -85,46 +94,100 @@ TEST(ThreadSafeArenaStatsTest, PrepareForSampling) { MutexLock l(&info.init_mu); info.PrepareForSampling(kTestStride); - EXPECT_EQ(info.num_allocations.load(), 0); - EXPECT_EQ(info.bytes_used.load(), 0); - EXPECT_EQ(info.bytes_allocated.load(), 0); - EXPECT_EQ(info.bytes_wasted.load(), 0); - EXPECT_EQ(info.max_block_size.load(), 0); + for (const auto& block_stats : info.block_histogram) { + EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0); + } + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0); EXPECT_EQ(info.weight, kTestStride); - info.num_allocations.store(1, std::memory_order_relaxed); - info.bytes_used.store(1, std::memory_order_relaxed); - info.bytes_allocated.store(1, std::memory_order_relaxed); - info.bytes_wasted.store(1, std::memory_order_relaxed); + for (auto& block_stats : info.block_histogram) { + block_stats.num_allocations.store(1, std::memory_order_relaxed); + block_stats.bytes_used.store(1, std::memory_order_relaxed); + block_stats.bytes_allocated.store(1, std::memory_order_relaxed); + block_stats.bytes_wasted.store(1, std::memory_order_relaxed); + } info.max_block_size.store(1, std::memory_order_relaxed); info.PrepareForSampling(2 * kTestStride); - EXPECT_EQ(info.num_allocations.load(), 0); - EXPECT_EQ(info.bytes_used.load(), 0); - EXPECT_EQ(info.bytes_allocated.load(), 0); - EXPECT_EQ(info.bytes_wasted.load(), 0); - EXPECT_EQ(info.max_block_size.load(), 0); + for (auto& block_stats : info.block_histogram) { + EXPECT_EQ(block_stats.num_allocations.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_used.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_allocated.load(std::memory_order_relaxed), 0); + EXPECT_EQ(block_stats.bytes_wasted.load(std::memory_order_relaxed), 0); + } + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 0); EXPECT_EQ(info.weight, 2 * kTestStride); } +TEST(ThreadSafeArenaStatsTest, FindBin) { + size_t current_bin = 0; + size_t bytes = 1; + while (current_bin < ThreadSafeArenaStats::kBlockHistogramBins - 1) { + size_t next_bin = ThreadSafeArenaStats::FindBin(bytes); + if (next_bin != current_bin) { + // Test the bins increase linearly. + EXPECT_EQ(next_bin, current_bin + 1); + // Test the bins change only at values of the form 2^k + 1. + EXPECT_EQ(absl::popcount(bytes - 1), 1); + current_bin = next_bin; + } + ++bytes; + } +} + +TEST(ThreadSafeArenaStatsTest, MinMaxBlockSizeForBin) { + std::pair current_limits = + ThreadSafeArenaStats::MinMaxBlockSizeForBin(0); + EXPECT_EQ(current_limits.first, 1); + EXPECT_LT(current_limits.first, current_limits.second); + for (size_t i = 1; i < ThreadSafeArenaStats::kBlockHistogramBins; ++i) { + std::pair next_limits = + ThreadSafeArenaStats::MinMaxBlockSizeForBin(i); + EXPECT_LT(next_limits.first, next_limits.second); + // Test the limits do not have gaps. + EXPECT_EQ(next_limits.first, current_limits.second + 1); + if (i != ThreadSafeArenaStats::kBlockHistogramBins - 1) { + EXPECT_EQ(next_limits.second, 2 * current_limits.second); + } + current_limits = next_limits; + } + // Test the limits cover the entire range possible. + EXPECT_EQ(current_limits.second, std::numeric_limits::max()); +} + TEST(ThreadSafeArenaStatsTest, RecordAllocateSlow) { ThreadSafeArenaStats info; constexpr int64_t kTestStride = 458; MutexLock l(&info.init_mu); info.PrepareForSampling(kTestStride); - RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); - EXPECT_EQ(info.num_allocations.load(), 1); - EXPECT_EQ(info.bytes_used.load(), 100); - EXPECT_EQ(info.bytes_allocated.load(), 128); - EXPECT_EQ(info.bytes_wasted.load(), 0); - EXPECT_EQ(info.max_block_size.load(), 128); + RecordAllocateSlow(&info, /*requested=*/0, /*allocated=*/128, /*wasted=*/0); + EXPECT_EQ( + info.block_histogram[0].num_allocations.load(std::memory_order_relaxed), + 1); + EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed), + 0); + EXPECT_EQ( + info.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed), + 128); + EXPECT_EQ( + info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 0); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256, /*wasted=*/28); - EXPECT_EQ(info.num_allocations.load(), 2); - EXPECT_EQ(info.bytes_used.load(), 200); - EXPECT_EQ(info.bytes_allocated.load(), 384); - EXPECT_EQ(info.bytes_wasted.load(), 28); - EXPECT_EQ(info.max_block_size.load(), 256); + EXPECT_EQ(info.block_histogram[0].bytes_used.load(std::memory_order_relaxed), + 100); + EXPECT_EQ( + info.block_histogram[0].bytes_wasted.load(std::memory_order_relaxed), 28); + EXPECT_EQ( + info.block_histogram[1].num_allocations.load(std::memory_order_relaxed), + 1); + EXPECT_EQ( + info.block_histogram[1].bytes_allocated.load(std::memory_order_relaxed), + 256); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256); } TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) { @@ -133,13 +196,13 @@ TEST(ThreadSafeArenaStatsTest, RecordAllocateSlowMaxBlockSizeTest) { MutexLock l(&info.init_mu); info.PrepareForSampling(kTestStride); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/0); - EXPECT_EQ(info.max_block_size.load(), 128); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 128); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/256, /*wasted=*/28); - EXPECT_EQ(info.max_block_size.load(), 256); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256); RecordAllocateSlow(&info, /*requested=*/100, /*allocated=*/128, /*wasted=*/28); - EXPECT_EQ(info.max_block_size.load(), 256); + EXPECT_EQ(info.max_block_size.load(std::memory_order_relaxed), 256); } TEST(ThreadSafeArenazSamplerTest, SamplingCorrectness) { @@ -212,12 +275,15 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { constexpr int64_t kTestStride = 17; ThreadSafeArenaStatsHandle h(sampler.Register(kTestStride)); auto* info = ThreadSafeArenaStatsHandlePeer::GetInfo(&h); - info->bytes_allocated.store(0x12345678, std::memory_order_relaxed); + info->block_histogram[0].bytes_allocated.store(0x12345678, + std::memory_order_relaxed); bool found = false; sampler.Iterate([&](const ThreadSafeArenaStats& h) { if (&h == info) { - EXPECT_EQ(h.bytes_allocated.load(), 0x12345678); + EXPECT_EQ( + h.block_histogram[0].bytes_allocated.load(std::memory_order_relaxed), + 0x12345678); EXPECT_EQ(h.weight, kTestStride); found = true; } @@ -230,7 +296,8 @@ TEST(ThreadSafeArenazSamplerTest, Handle) { if (&h == info) { // this will only happen if some other thread has resurrected the info // the old handle was using. - if (h.bytes_allocated.load() == 0x12345678) { + if (h.block_histogram[0].bytes_allocated.load( + std::memory_order_relaxed) == 0x12345678) { found = true; } } @@ -246,7 +313,7 @@ TEST(ThreadSafeArenazSamplerTest, Registration) { auto* info2 = Register(&sampler, 2, kTestStride); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2)); - info1->bytes_allocated.store(3); + info1->block_histogram[0].bytes_allocated.store(3, std::memory_order_relaxed); EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(3, 2)); sampler.Unregister(info1); @@ -258,18 +325,18 @@ TEST(ThreadSafeArenazSamplerTest, Unregistration) { std::vector infos; constexpr int64_t kTestStride = 200; for (size_t i = 0; i < 3; ++i) { - infos.push_back(Register(&sampler, i, kTestStride)); + infos.push_back(Register(&sampler, i + 1, kTestStride)); } - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 1, 2)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 2, 3)); sampler.Unregister(infos[1]); - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3)); infos.push_back(Register(&sampler, 3, kTestStride)); infos.push_back(Register(&sampler, 4, kTestStride)); - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 3, 4)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 3, 4)); sampler.Unregister(infos[3]); - EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(0, 2, 4)); + EXPECT_THAT(GetBytesAllocated(&sampler), UnorderedElementsAre(1, 3, 4)); sampler.Unregister(infos[0]); sampler.Unregister(infos[2]); diff --git a/src/google/protobuf/compiler/command_line_interface.cc b/src/google/protobuf/compiler/command_line_interface.cc index 74910461ec..f5cc9a9173 100644 --- a/src/google/protobuf/compiler/command_line_interface.cc +++ b/src/google/protobuf/compiler/command_line_interface.cc @@ -2124,7 +2124,8 @@ bool CommandLineInterface::EnforceProto3OptionalSupport( << codegen_name << " hasn't been updated to support optional fields in " "proto3. Please ask the owner of this code generator to " - "support proto3 optional."; + "support proto3 optional." + << std::endl; return false; } } diff --git a/src/google/protobuf/compiler/cpp/options.h b/src/google/protobuf/compiler/cpp/options.h index 88d4b3097a..7aab669131 100644 --- a/src/google/protobuf/compiler/cpp/options.h +++ b/src/google/protobuf/compiler/cpp/options.h @@ -85,7 +85,7 @@ struct Options { bool profile_driven_inline_string = true; bool message_owned_arena_trial = false; bool force_split = false; - bool profile_driven_split = false; + bool profile_driven_split = true; #ifdef PROTOBUF_STABLE_EXPERIMENTS bool force_eagerly_verified_lazy = true; bool force_inline_string = true; diff --git a/src/google/protobuf/compiler/java/file.cc b/src/google/protobuf/compiler/java/file.cc index bd3647d7b8..c4665182d4 100644 --- a/src/google/protobuf/compiler/java/file.cc +++ b/src/google/protobuf/compiler/java/file.cc @@ -704,10 +704,13 @@ void FileGenerator::GenerateKotlinSiblings( options_.annotate_code ? &annotation_collector : nullptr); printer.Print( - "//Generated by the protocol buffer compiler. DO NOT EDIT!\n" + "// Generated by the protocol buffer compiler. DO NOT EDIT!\n" "// source: $filename$\n" "\n", "filename", descriptor->file()->name()); + printer.Print( + "// Generated files should ignore deprecation warnings\n" + "@file:Suppress(\"DEPRECATION\")"); if (!java_package_.empty()) { printer.Print( "package $package$;\n" diff --git a/src/google/protobuf/compiler/python/generator.cc b/src/google/protobuf/compiler/python/generator.cc index d8d6d74923..70b08042bb 100644 --- a/src/google/protobuf/compiler/python/generator.cc +++ b/src/google/protobuf/compiler/python/generator.cc @@ -217,15 +217,12 @@ bool Generator::Generate(const FileDescriptor* file, GeneratorContext* context, std::string* error) const { // ----------------------------------------------------------------- // parse generator options - bool cpp_generated_lib_linked = false; std::vector > options; ParseGeneratorParameter(parameter, &options); for (int i = 0; i < options.size(); i++) { - if (options[i].first == "cpp_generated_lib_linked") { - cpp_generated_lib_linked = true; - } else if (options[i].first == "pyi_out") { + if (options[i].first == "pyi_out") { python::PyiGenerator pyi_generator; if (!pyi_generator.Generate(file, "", context, error)) { return false; @@ -247,10 +244,6 @@ bool Generator::Generate(const FileDescriptor* file, file_ = file; std::string filename = GetFileName(file, ".py"); - pure_python_workable_ = !cpp_generated_lib_linked; - if (HasPrefixString(file->name(), "google/protobuf/")) { - pure_python_workable_ = true; - } FileDescriptorProto fdp; file_->CopyTo(&fdp); @@ -263,55 +256,49 @@ bool Generator::Generate(const FileDescriptor* file, printer_ = &printer; PrintTopBoilerplate(printer_, file_, GeneratingDescriptorProto()); - if (pure_python_workable_) { - PrintImports(); - } + PrintImports(); PrintFileDescriptor(); - if (pure_python_workable_) { - if (GeneratingDescriptorProto()) { - printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); - printer_->Indent(); - // Create enums before message descriptors - PrintAllNestedEnumsInFile(); - PrintMessageDescriptors(); - FixForeignFieldsInDescriptors(); - printer_->Outdent(); - printer_->Print("else:\n"); - printer_->Indent(); - } - // Find the message descriptors first and then use the message - // descriptor to find enums. - printer_->Print( - "_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n"); - if (GeneratingDescriptorProto()) { - printer_->Outdent(); - } + if (GeneratingDescriptorProto()) { + printer_->Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); + printer_->Indent(); + // Create enums before message descriptors + PrintAllNestedEnumsInFile(); + PrintMessageDescriptors(); + FixForeignFieldsInDescriptors(); + printer_->Outdent(); + printer_->Print("else:\n"); + printer_->Indent(); + } + // Find the message descriptors first and then use the message + // descriptor to find enums. + printer_->Print( + "_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())\n"); + if (GeneratingDescriptorProto()) { + printer_->Outdent(); } std::string module_name = ModuleName(file->name()); printer_->Print( "_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, '$module_name$', " "globals())\n", "module_name", module_name); - if (pure_python_workable_) { - printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); - printer_->Indent(); + printer.Print("if _descriptor._USE_C_DESCRIPTORS == False:\n"); + printer_->Indent(); - // We have to fix up the extensions after the message classes themselves, - // since they need to call static RegisterExtension() methods on these - // classes. - FixForeignFieldsInExtensions(); - // Descriptor options may have custom extensions. These custom options - // can only be successfully parsed after we register corresponding - // extensions. Therefore we parse all options again here to recognize - // custom options that may be unknown when we define the descriptors. - // This does not apply to services because they are not used by extensions. - FixAllDescriptorOptions(); + // We have to fix up the extensions after the message classes themselves, + // since they need to call static RegisterExtension() methods on these + // classes. + FixForeignFieldsInExtensions(); + // Descriptor options may have custom extensions. These custom options + // can only be successfully parsed after we register corresponding + // extensions. Therefore we parse all options again here to recognize + // custom options that may be unknown when we define the descriptors. + // This does not apply to services because they are not used by extensions. + FixAllDescriptorOptions(); - // Set serialized_start and serialized_end. - SetSerializedPbInterval(); + // Set serialized_start and serialized_end. + SetSerializedPbInterval(); - printer_->Outdent(); - } + printer_->Outdent(); if (HasGenericServices(file)) { printer_->Print( "_builder.BuildServices(DESCRIPTOR, '$module_name$', globals())\n", @@ -389,30 +376,26 @@ void Generator::PrintFileDescriptor() const { " create_key=_descriptor._internal_create_key,\n"; printer_->Print(m, file_descriptor_template); printer_->Indent(); - if (pure_python_workable_) { - printer_->Print("serialized_pb=b'$value$'\n", "value", - strings::CHexEscape(file_descriptor_serialized_)); - if (file_->dependency_count() != 0) { - printer_->Print(",\ndependencies=["); - for (int i = 0; i < file_->dependency_count(); ++i) { - std::string module_alias = ModuleAlias(file_->dependency(i)->name()); - printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", - module_alias); - } - printer_->Print("]"); + printer_->Print("serialized_pb=b'$value$'\n", "value", + strings::CHexEscape(file_descriptor_serialized_)); + if (file_->dependency_count() != 0) { + printer_->Print(",\ndependencies=["); + for (int i = 0; i < file_->dependency_count(); ++i) { + std::string module_alias = ModuleAlias(file_->dependency(i)->name()); + printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", + module_alias); } - if (file_->public_dependency_count() > 0) { - printer_->Print(",\npublic_dependencies=["); - for (int i = 0; i < file_->public_dependency_count(); ++i) { - std::string module_alias = - ModuleAlias(file_->public_dependency(i)->name()); - printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", - module_alias); - } - printer_->Print("]"); + printer_->Print("]"); + } + if (file_->public_dependency_count() > 0) { + printer_->Print(",\npublic_dependencies=["); + for (int i = 0; i < file_->public_dependency_count(); ++i) { + std::string module_alias = + ModuleAlias(file_->public_dependency(i)->name()); + printer_->Print("$module_alias$.DESCRIPTOR,", "module_alias", + module_alias); } - } else { - printer_->Print("serialized_pb=''\n"); + printer_->Print("]"); } // TODO(falk): Also print options and fix the message_type, enum_type, @@ -467,11 +450,9 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { printer_->Indent(); printer_->Indent(); - if (pure_python_workable_) { - for (int i = 0; i < enum_descriptor.value_count(); ++i) { - PrintEnumValueDescriptor(*enum_descriptor.value(i)); - printer_->Print(",\n"); - } + for (int i = 0; i < enum_descriptor.value_count(); ++i) { + PrintEnumValueDescriptor(*enum_descriptor.value(i)); + printer_->Print(",\n"); } printer_->Outdent(); @@ -482,10 +463,8 @@ void Generator::PrintEnum(const EnumDescriptor& enum_descriptor) const { EnumDescriptorProto edp; printer_->Outdent(); printer_->Print(")\n"); - if (pure_python_workable_) { - printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name", - module_level_descriptor_name); - } + printer_->Print("_sym_db.RegisterEnumDescriptor($name$)\n", "name", + module_level_descriptor_name); printer_->Print("\n"); } @@ -535,10 +514,6 @@ void Generator::PrintServiceDescriptor( void Generator::PrintDescriptorKeyAndModuleName( const ServiceDescriptor& descriptor) const { std::string name = ModuleLevelServiceDescriptorName(descriptor); - if (!pure_python_workable_) { - name = "_descriptor.ServiceDescriptor(full_name='" + - descriptor.full_name() + "')"; - } printer_->Print("$descriptor_key$ = $descriptor_name$,\n", "descriptor_key", kDescriptorKey, "descriptor_name", name); std::string module_name = ModuleName(file_->name()); @@ -728,12 +703,7 @@ void Generator::PrintMessage(const Descriptor& message_descriptor, PrintNestedMessages(message_descriptor, qualified_name, to_register); std::map m; m["descriptor_key"] = kDescriptorKey; - if (pure_python_workable_) { - m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor); - } else { - m["descriptor_name"] = "_descriptor.Descriptor(full_name='" + - message_descriptor.full_name() + "')"; - } + m["descriptor_name"] = ModuleLevelDescriptorName(message_descriptor); printer_->Print(m, "'$descriptor_key$' : $descriptor_name$,\n"); std::string module_name = ModuleName(file_->name()); printer_->Print("'__module__' : '$module_name$'\n", "module_name", diff --git a/src/google/protobuf/compiler/python/generator.h b/src/google/protobuf/compiler/python/generator.h index 0b8d56de70..46b0ad1836 100644 --- a/src/google/protobuf/compiler/python/generator.h +++ b/src/google/protobuf/compiler/python/generator.h @@ -174,7 +174,6 @@ class PROTOC_EXPORT Generator : public CodeGenerator { mutable const FileDescriptor* file_; // Set in Generate(). Under mutex_. mutable std::string file_descriptor_serialized_; mutable io::Printer* printer_; // Set in Generate(). Under mutex_. - mutable bool pure_python_workable_; bool opensource_runtime_ = true; diff --git a/src/google/protobuf/descriptor.h b/src/google/protobuf/descriptor.h index 685e9235d8..beb3e641c6 100644 --- a/src/google/protobuf/descriptor.h +++ b/src/google/protobuf/descriptor.h @@ -1577,7 +1577,11 @@ class PROTOBUF_EXPORT FileDescriptor : private internal::SymbolBase { const FileOptions& options() const; // Syntax of this file. - enum Syntax { + enum Syntax +#ifndef SWIG + : int +#endif // !SWIG + { SYNTAX_UNKNOWN = 0, SYNTAX_PROTO2 = 2, SYNTAX_PROTO3 = 3, diff --git a/src/google/protobuf/message.h b/src/google/protobuf/message.h index 5a9111f6a1..0d69a6bc38 100644 --- a/src/google/protobuf/message.h +++ b/src/google/protobuf/message.h @@ -485,6 +485,11 @@ class PROTOBUF_EXPORT Reflection final { return internal::ToIntSize(SpaceUsedLong(message)); } + // Returns true if the given message is a default message instance. + bool IsDefaultInstance(const Message& message) const { + return schema_.IsDefaultInstance(message); + } + // Check if the given non-repeated field is set. bool HasField(const Message& message, const FieldDescriptor* field) const; diff --git a/src/google/protobuf/message_unittest.inc b/src/google/protobuf/message_unittest.inc index a1b9202ffe..c7c29e6cd9 100644 --- a/src/google/protobuf/message_unittest.inc +++ b/src/google/protobuf/message_unittest.inc @@ -1343,5 +1343,13 @@ TEST(MESSAGE_TEST_NAME, TestBoolParsers) { } } +TEST(MESSAGE_TEST_NAME, IsDefaultInstance) { + UNITTEST::TestAllTypes msg; + const auto& default_msg = UNITTEST::TestAllTypes::default_instance(); + const auto* r = msg.GetReflection(); + EXPECT_TRUE(r->IsDefaultInstance(default_msg)); + EXPECT_FALSE(r->IsDefaultInstance(msg)); +} + } // namespace protobuf } // namespace google diff --git a/src/google/protobuf/repeated_field.h b/src/google/protobuf/repeated_field.h index b4e441681f..a7c1275b60 100644 --- a/src/google/protobuf/repeated_field.h +++ b/src/google/protobuf/repeated_field.h @@ -94,43 +94,30 @@ constexpr int RepeatedFieldLowerClampLimit() { constexpr int kRepeatedFieldUpperClampLimit = (std::numeric_limits::max() / 2) + 1; -// Swaps two blocks of memory of size sizeof(T). -template -inline void SwapBlock(char* p, char* q) { - T tmp; - memcpy(&tmp, p, sizeof(T)); - memcpy(p, q, sizeof(T)); - memcpy(q, &tmp, sizeof(T)); -} - // Swaps two blocks of memory of size kSize: -// template void memswap(char* p, char* q); -template -inline typename std::enable_if<(kSize == 0), void>::type memswap(char*, char*) { -} - -#define PROTO_MEMSWAP_DEF_SIZE(reg_type, max_size) \ - template \ - typename std::enable_if<(kSize >= sizeof(reg_type) && kSize < (max_size)), \ - void>::type \ - memswap(char* p, char* q) { \ - SwapBlock(p, q); \ - memswap(p + sizeof(reg_type), \ - q + sizeof(reg_type)); \ - } - -PROTO_MEMSWAP_DEF_SIZE(uint8_t, 2) -PROTO_MEMSWAP_DEF_SIZE(uint16_t, 4) -PROTO_MEMSWAP_DEF_SIZE(uint32_t, 8) - -#ifdef __SIZEOF_INT128__ -PROTO_MEMSWAP_DEF_SIZE(uint64_t, 16) -PROTO_MEMSWAP_DEF_SIZE(__uint128_t, (1u << 31)) +template +void memswap(char* a, char* b) { +#if __SIZEOF_INT128__ + using Buffer = __uint128_t; #else -PROTO_MEMSWAP_DEF_SIZE(uint64_t, (1u << 31)) + using Buffer = uint64_t; #endif -#undef PROTO_MEMSWAP_DEF_SIZE + constexpr size_t kBlockSize = sizeof(Buffer); + Buffer buf; + for (size_t i = 0; i < kSize / kBlockSize; ++i) { + memcpy(&buf, a, kBlockSize); + memcpy(a, b, kBlockSize); + memcpy(b, &buf, kBlockSize); + a += kBlockSize; + b += kBlockSize; + } + + // Swap the leftover bytes, could be zero. + memcpy(&buf, a, kSize % kBlockSize); + memcpy(a, b, kSize % kBlockSize); + memcpy(b, &buf, kSize % kBlockSize); +} template class RepeatedIterator; diff --git a/src/google/protobuf/repeated_field_unittest.cc b/src/google/protobuf/repeated_field_unittest.cc index c5211470bc..2cb6bab8c7 100644 --- a/src/google/protobuf/repeated_field_unittest.cc +++ b/src/google/protobuf/repeated_field_unittest.cc @@ -305,6 +305,60 @@ TEST(RepeatedField, SwapLargeLarge) { } } +template +void TestMemswap() { + SCOPED_TRACE(kSize); + + const auto a_char = [](int i) -> char { return (i % ('z' - 'a')) + 'a'; }; + const auto b_char = [](int i) -> char { return (i % ('Z' - 'A')) + 'A'; }; + std::string a, b; + for (int i = 0; i < kSize; ++i) { + a += a_char(i); + b += b_char(i); + } + // We will not swap these. + a += "+"; + b += "-"; + + std::string expected_a = b, expected_b = a; + expected_a.back() = '+'; + expected_b.back() = '-'; + + internal::memswap(&a[0], &b[0]); + + // ODR use the functions in a way that forces the linker to keep them. That + // way we can see their generated code. + volatile auto odr_use_for_asm_dump = &internal::memswap; + (void)odr_use_for_asm_dump; + + EXPECT_EQ(expected_a, a); + EXPECT_EQ(expected_b, b); +} + +TEST(Memswap, VerifyWithSmallAndLargeSizes) { + // Arbitrary sizes + TestMemswap<0>(); + TestMemswap<1>(); + TestMemswap<10>(); + TestMemswap<100>(); + TestMemswap<1000>(); + TestMemswap<10000>(); + TestMemswap<100000>(); + TestMemswap<1000000>(); + + // Pointer aligned sizes + TestMemswap(); + TestMemswap(); + TestMemswap(); + TestMemswap(); + + // Test also just the block size and no leftover. + TestMemswap<64 * 1>(); + TestMemswap<64 * 2>(); + TestMemswap<64 * 3>(); + TestMemswap<64 * 4>(); +} + // Determines how much space was reserved by the given field by adding elements // to it until it re-allocates its space. static int ReservedSpace(RepeatedField* field) {