From 5330a36c15901901ad7d1476af5b7658c4304ca4 Mon Sep 17 00:00:00 2001 From: Protobuf Team Bot Date: Tue, 10 Jan 2023 10:13:17 -0800 Subject: [PATCH] Add a new test input stream `TestZeroCopyInputStream` for more control over the particular buffers passed in tests. PiperOrigin-RevId: 501027379 --- src/google/protobuf/io/BUILD.bazel | 26 +++ src/google/protobuf/io/coded_stream.h | 2 +- .../protobuf/io/test_zero_copy_stream.h | 133 +++++++++++ .../protobuf/io/test_zero_copy_stream_test.cc | 215 ++++++++++++++++++ 4 files changed, 375 insertions(+), 1 deletion(-) create mode 100644 src/google/protobuf/io/test_zero_copy_stream.h create mode 100644 src/google/protobuf/io/test_zero_copy_stream_test.cc diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel index 88f1ee1bf6..e61a2a26a7 100644 --- a/src/google/protobuf/io/BUILD.bazel +++ b/src/google/protobuf/io/BUILD.bazel @@ -32,6 +32,32 @@ cc_library( ], ) +cc_library( + name = "test_zero_copy_stream", + testonly = 1, + hdrs = ["test_zero_copy_stream.h"], + copts = COPTS, + include_prefix = "google/protobuf/io", + deps = [ + ":io", + "//src/google/protobuf/stubs", + "@com_google_absl//absl/types:optional", + ], +) + +cc_test( + name = "test_zero_copy_stream_test", + srcs = ["test_zero_copy_stream_test.cc"], + copts = COPTS, + deps = [ + ":test_zero_copy_stream", + "//src/google/protobuf/stubs", + "@com_google_absl//absl/types:optional", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "zero_copy_sink", srcs = ["zero_copy_sink.cc"], diff --git a/src/google/protobuf/io/coded_stream.h b/src/google/protobuf/io/coded_stream.h index 98ff721f0f..fc4c55c978 100644 --- a/src/google/protobuf/io/coded_stream.h +++ b/src/google/protobuf/io/coded_stream.h @@ -1292,7 +1292,7 @@ class PROTOBUF_EXPORT CodedOutputStream { // that wants deterministic serialization by default needs to call // SetDefaultSerializationDeterministic() or ensure on its own that another // thread has done so. - friend void internal::MapTestForceDeterministic(); + friend void google::protobuf::internal::MapTestForceDeterministic(); static void SetDefaultSerializationDeterministic() { default_serialization_deterministic_.store(true, std::memory_order_relaxed); } diff --git a/src/google/protobuf/io/test_zero_copy_stream.h b/src/google/protobuf/io/test_zero_copy_stream.h new file mode 100644 index 0000000000..b03b10b0c7 --- /dev/null +++ b/src/google/protobuf/io/test_zero_copy_stream.h @@ -0,0 +1,133 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef GOOGLE_PROTOBUF_IO_TEST_ZERO_COPY_STREAM_H__ +#define GOOGLE_PROTOBUF_IO_TEST_ZERO_COPY_STREAM_H__ + +#include +#include +#include +#include + +#include "google/protobuf/stubs/logging.h" +#include "absl/types/optional.h" +#include "google/protobuf/io/zero_copy_stream.h" + +// Must be included last. +#include "google/protobuf/port_def.inc" + +namespace google { +namespace protobuf { +namespace io { +namespace internal { + +// Input stream used for testing the proper handling of input fragmentation. +// It also will assert the preconditions of the methods. +class TestZeroCopyInputStream final : public ZeroCopyInputStream { + public: + // The input stream will provide the buffers exactly as passed here. + explicit TestZeroCopyInputStream(const std::vector& buffers) + : buffers_(buffers.begin(), buffers.end()) {} + + // Allow copy to be able to inspect the stream without consuming the original. + TestZeroCopyInputStream(const TestZeroCopyInputStream& other) + : ZeroCopyInputStream(), + buffers_(other.buffers_), + last_returned_buffer_(other.last_returned_buffer_), + byte_count_(other.byte_count_) {} + + bool Next(const void** data, int* size) override { + GOOGLE_ABSL_CHECK(data) << "data must not be null"; + GOOGLE_ABSL_CHECK(size) << "size must not be null"; + last_returned_buffer_ = absl::nullopt; + + // We are done + if (buffers_.empty()) return false; + + last_returned_buffer_ = std::move(buffers_.front()); + buffers_.pop_front(); + *data = last_returned_buffer_->data(); + *size = static_cast(last_returned_buffer_->size()); + byte_count_ += *size; + return true; + } + + void BackUp(int count) override { + GOOGLE_ABSL_CHECK_GE(count, 0) << "count must not be negative"; + GOOGLE_ABSL_CHECK(last_returned_buffer_.has_value()) + << "The last call was not a successful Next()"; + GOOGLE_ABSL_CHECK_LE(count, last_returned_buffer_->size()) + << "count must be within bounds of last buffer"; + buffers_.push_front( + last_returned_buffer_->substr(last_returned_buffer_->size() - count)); + last_returned_buffer_ = absl::nullopt; + byte_count_ -= count; + } + + bool Skip(int count) override { + GOOGLE_ABSL_CHECK_GE(count, 0) << "count must not be negative"; + last_returned_buffer_ = absl::nullopt; + while (true) { + if (count == 0) return true; + if (buffers_.empty()) return false; + auto& front = buffers_[0]; + int front_size = static_cast(front.size()); + if (front_size <= count) { + count -= front_size; + byte_count_ += front_size; + buffers_.pop_front(); + } else { + // The front is enough, just chomp from it. + front = front.substr(count); + byte_count_ += count; + return true; + } + } + } + + int64_t ByteCount() const override { return byte_count_; } + + private: + // For simplicity of implementation, we pop the elements from the from and + // move them to `last_returned_buffer_`. It makes it simpler to keep track of + // the state of the object. The extra cost is not relevant for testing. + std::deque buffers_; + absl::optional last_returned_buffer_; + int64_t byte_count_ = 0; +}; + +} // namespace internal +} // namespace io +} // namespace protobuf +} // namespace google + +#include "google/protobuf/port_undef.inc" + +#endif // GOOGLE_PROTOBUF_IO_TEST_ZERO_COPY_STREAM_H__ diff --git a/src/google/protobuf/io/test_zero_copy_stream_test.cc b/src/google/protobuf/io/test_zero_copy_stream_test.cc new file mode 100644 index 0000000000..0e5e59e41c --- /dev/null +++ b/src/google/protobuf/io/test_zero_copy_stream_test.cc @@ -0,0 +1,215 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2023 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "google/protobuf/io/test_zero_copy_stream.h" + +#include +#include +#include +#include + +#include +#include +#include "absl/types/optional.h" + +namespace google { +namespace protobuf { +namespace io { +namespace internal { +namespace { + +using testing::ElementsAre; +using testing::Eq; +using testing::Optional; + +absl::optional CallNext(ZeroCopyInputStream& stream) { + const void* data; + int size; + if (stream.Next(&data, &size)) { + return std::string(static_cast(data), + static_cast(size)); + } + return absl::nullopt; +} + +std::vector ReadLeftoverDoNotConsumeInput( + TestZeroCopyInputStream copy) { + std::vector out; + while (auto next = CallNext(copy)) { + out.push_back(*std::move(next)); + } + return out; +} + +#if PROTOBUF_HAS_DEATH_TEST +TEST(TestZeroCopyInputStreamTest, NextChecksPreconditions) { + std::unique_ptr stream = + std::make_unique(std::vector{}); + const void* data; + int size; + EXPECT_DEATH(stream->Next(nullptr, &size), "data must not be null"); + EXPECT_DEATH(stream->Next(&data, nullptr), "size must not be null"); +} +#endif // PROTOBUF_HAS_DEATH_TEST + +TEST(TestZeroCopyInputStreamTest, NextProvidesTheBuffersCorrectly) { + std::vector expected = {"ABC", "D", "EFG", "", "", "HIJKLMN"}; + std::unique_ptr stream = + std::make_unique(expected); + + std::vector found; + while (auto next = CallNext(*stream)) { + found.push_back(*std::move(next)); + } + + EXPECT_EQ(found, expected); +} + +TEST(TestZeroCopyInputStreamTest, BackUpGivesBackABuffer) { + std::vector expected = {"ABC", "D", "EFG", "", "", "HIJKLMN"}; + std::unique_ptr stream = + std::make_unique(expected); + + EXPECT_THAT(CallNext(*stream), Optional(Eq("ABC"))); + stream->BackUp(3); + EXPECT_THAT(CallNext(*stream), Optional(Eq("ABC"))); + stream->BackUp(2); + EXPECT_THAT(CallNext(*stream), Optional(Eq("BC"))); + EXPECT_THAT(CallNext(*stream), Optional(Eq("D"))); + stream->BackUp(1); + EXPECT_THAT(CallNext(*stream), Optional(Eq("D"))); + stream->BackUp(0); + EXPECT_THAT(CallNext(*stream), Optional(Eq(""))); + EXPECT_THAT(CallNext(*stream), Optional(Eq("EFG"))); + EXPECT_THAT(CallNext(*stream), Optional(Eq(""))); + EXPECT_THAT(CallNext(*stream), Optional(Eq(""))); + EXPECT_THAT(CallNext(*stream), Optional(Eq("HIJKLMN"))); + stream->BackUp(2); + EXPECT_THAT(CallNext(*stream), Optional(Eq("MN"))); + EXPECT_THAT(CallNext(*stream), Eq(absl::nullopt)); +} + +#if PROTOBUF_HAS_DEATH_TEST +TEST(TestZeroCopyInputStreamTest, BackUpChecksPreconditions) { + std::vector expected = {"ABC", "D", "EFG", "", "", "HIJKLMN"}; + std::unique_ptr stream = + std::make_unique(expected); + + EXPECT_DEATH(stream->BackUp(0), "The last call was not a successful Next()"); + EXPECT_THAT(CallNext(*stream), Optional(Eq("ABC"))); + EXPECT_DEATH(stream->BackUp(-1), "count must not be negative"); + stream->BackUp(1); + EXPECT_DEATH(stream->BackUp(0), "The last call was not a successful Next()"); + EXPECT_THAT(CallNext(*stream), Optional(Eq("C"))); + EXPECT_THAT(CallNext(*stream), Optional(Eq("D"))); + stream->Skip(1); + EXPECT_DEATH(stream->BackUp(0), "The last call was not a successful Next()"); + EXPECT_THAT(CallNext(*stream), Optional(Eq("FG"))); + EXPECT_THAT(CallNext(*stream), Optional(Eq(""))); + EXPECT_THAT(CallNext(*stream), Optional(Eq(""))); + EXPECT_THAT(CallNext(*stream), Optional(Eq("HIJKLMN"))); + EXPECT_DEATH(stream->BackUp(8), "count must be within bounds of last buffer"); + EXPECT_THAT(CallNext(*stream), Eq(absl::nullopt)); + EXPECT_DEATH(stream->BackUp(0), "The last call was not a successful Next()"); +} +#endif // PROTOBUF_HAS_DEATH_TEST + +TEST(TestZeroCopyInputStreamTest, SkipWorks) { + std::vector expected = {"ABC", "D", "EFG", "", "", "HIJKLMN"}; + TestZeroCopyInputStream stream(expected); + + EXPECT_THAT(ReadLeftoverDoNotConsumeInput(stream), + ElementsAre("ABC", "D", "EFG", "", "", "HIJKLMN")); + // Skip nothing + EXPECT_TRUE(stream.Skip(0)); + EXPECT_THAT(ReadLeftoverDoNotConsumeInput(stream), + ElementsAre("ABC", "D", "EFG", "", "", "HIJKLMN")); + // Skip less than one chunk + EXPECT_TRUE(stream.Skip(1)); + EXPECT_THAT(ReadLeftoverDoNotConsumeInput(stream), + ElementsAre("BC", "D", "EFG", "", "", "HIJKLMN")); + // Skip exactly one chunk + EXPECT_TRUE(stream.Skip(2)); + EXPECT_THAT(ReadLeftoverDoNotConsumeInput(stream), + ElementsAre("D", "EFG", "", "", "HIJKLMN")); + // Skip cross chunks + EXPECT_TRUE(stream.Skip(3)); + EXPECT_THAT(ReadLeftoverDoNotConsumeInput(stream), + ElementsAre("G", "", "", "HIJKLMN")); + // Skip the rest + EXPECT_TRUE(stream.Skip(8)); + EXPECT_THAT(ReadLeftoverDoNotConsumeInput(stream), ElementsAre()); + // Skipping zero works on empty + EXPECT_TRUE(stream.Skip(0)); + // but skipping non-zero does not + EXPECT_FALSE(stream.Skip(1)); +} + +#if PROTOBUF_HAS_DEATH_TEST +TEST(TestZeroCopyInputStreamTest, SkipChecksPreconditions) { + std::unique_ptr stream = + std::make_unique(std::vector{}); + EXPECT_DEATH(stream->Skip(-1), "count must not be negative"); +} +#endif // PROTOBUF_HAS_DEATH_TEST + +TEST(TestZeroCopyInputStreamTest, ByteCountWorks) { + std::vector expected = {"ABC", "D", "EFG", "", "", "HIJKLMN"}; + TestZeroCopyInputStream stream(expected); + EXPECT_EQ(stream.ByteCount(), 0); + EXPECT_TRUE(stream.Skip(0)); + EXPECT_EQ(stream.ByteCount(), 0); + EXPECT_TRUE(stream.Skip(1)); + EXPECT_EQ(stream.ByteCount(), 1); + EXPECT_THAT(CallNext(stream), Optional(Eq("BC"))); + EXPECT_EQ(stream.ByteCount(), 3); + stream.BackUp(1); + EXPECT_EQ(stream.ByteCount(), 2); + EXPECT_THAT(CallNext(stream), Optional(Eq("C"))); + EXPECT_EQ(stream.ByteCount(), 3); + EXPECT_THAT(CallNext(stream), Optional(Eq("D"))); + EXPECT_EQ(stream.ByteCount(), 4); + EXPECT_THAT(CallNext(stream), Optional(Eq("EFG"))); + EXPECT_EQ(stream.ByteCount(), 7); + EXPECT_THAT(CallNext(stream), Optional(Eq(""))); + EXPECT_EQ(stream.ByteCount(), 7); + EXPECT_THAT(CallNext(stream), Optional(Eq(""))); + EXPECT_EQ(stream.ByteCount(), 7); + EXPECT_TRUE(stream.Skip(3)); + EXPECT_EQ(stream.ByteCount(), 10); + EXPECT_TRUE(stream.Skip(4)); + EXPECT_EQ(stream.ByteCount(), 14); +} + +} // namespace +} // namespace internal +} // namespace io +} // namespace protobuf +} // namespace google