diff --git a/BUILD b/BUILD index bf8946282f..56f9ea38f8 100644 --- a/BUILD +++ b/BUILD @@ -145,6 +145,7 @@ cc_library( "src/google/protobuf/timestamp.pb.cc", "src/google/protobuf/type.pb.cc", "src/google/protobuf/unknown_field_set.cc", + "src/google/protobuf/util/delimited_message_util.cc", "src/google/protobuf/util/field_comparator.cc", "src/google/protobuf/util/field_mask_util.cc", "src/google/protobuf/util/internal/datapiece.cc", @@ -521,6 +522,7 @@ cc_test( "src/google/protobuf/stubs/type_traits_unittest.cc", "src/google/protobuf/text_format_unittest.cc", "src/google/protobuf/unknown_field_set_unittest.cc", + "src/google/protobuf/util/delimited_message_util_test.cc", "src/google/protobuf/util/field_comparator_test.cc", "src/google/protobuf/util/field_mask_util_test.cc", "src/google/protobuf/util/internal/default_value_objectwriter_test.cc", diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake index 26e1f35617..5313d39ef8 100644 --- a/cmake/libprotobuf.cmake +++ b/cmake/libprotobuf.cmake @@ -30,6 +30,7 @@ set(libprotobuf_files ${protobuf_source_dir}/src/google/protobuf/timestamp.pb.cc ${protobuf_source_dir}/src/google/protobuf/type.pb.cc ${protobuf_source_dir}/src/google/protobuf/unknown_field_set.cc + ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util.cc ${protobuf_source_dir}/src/google/protobuf/util/field_comparator.cc ${protobuf_source_dir}/src/google/protobuf/util/field_mask_util.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/datapiece.cc diff --git a/cmake/tests.cmake b/cmake/tests.cmake index 978fd0c5b8..1470e4b036 100644 --- a/cmake/tests.cmake +++ b/cmake/tests.cmake @@ -169,6 +169,7 @@ set(tests_files ${protobuf_source_dir}/src/google/protobuf/stubs/type_traits_unittest.cc ${protobuf_source_dir}/src/google/protobuf/text_format_unittest.cc ${protobuf_source_dir}/src/google/protobuf/unknown_field_set_unittest.cc + ${protobuf_source_dir}/src/google/protobuf/util/delimited_message_util_test.cc ${protobuf_source_dir}/src/google/protobuf/util/field_comparator_test.cc ${protobuf_source_dir}/src/google/protobuf/util/field_mask_util_test.cc ${protobuf_source_dir}/src/google/protobuf/util/internal/default_value_objectwriter_test.cc diff --git a/src/Makefile.am b/src/Makefile.am index 6a91044da0..bcd81576b2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -169,6 +169,7 @@ nobase_include_HEADERS = \ google/protobuf/compiler/python/python_generator.h \ google/protobuf/compiler/ruby/ruby_generator.h \ google/protobuf/util/type_resolver.h \ + google/protobuf/util/delimited_message_util.h \ google/protobuf/util/field_comparator.h \ google/protobuf/util/field_mask_util.h \ google/protobuf/util/json_util.h \ @@ -267,6 +268,7 @@ libprotobuf_la_SOURCES = \ google/protobuf/io/zero_copy_stream_impl.cc \ google/protobuf/compiler/importer.cc \ google/protobuf/compiler/parser.cc \ + google/protobuf/util/delimited_message_util.cc \ google/protobuf/util/field_comparator.cc \ google/protobuf/util/field_mask_util.cc \ google/protobuf/util/internal/constants.h \ @@ -800,6 +802,7 @@ protobuf_test_SOURCES = \ google/protobuf/compiler/ruby/ruby_generator_unittest.cc \ google/protobuf/compiler/csharp/csharp_bootstrap_unittest.cc \ google/protobuf/compiler/csharp/csharp_generator_unittest.cc \ + google/protobuf/util/delimited_message_util_test.cc \ google/protobuf/util/field_comparator_test.cc \ google/protobuf/util/field_mask_util_test.cc \ google/protobuf/util/internal/default_value_objectwriter_test.cc \ diff --git a/src/google/protobuf/util/delimited_message_util.cc b/src/google/protobuf/util/delimited_message_util.cc new file mode 100644 index 0000000000..1637878216 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.cc @@ -0,0 +1,79 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include + +namespace google { +namespace protobuf { +namespace util { + +bool SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor) { + io::FileOutputStream output(file_descriptor); + return SerializeDelimitedToZeroCopyStream(message, &output); +} + +bool SerializeDelimitedToOstream(const MessageLite& message, ostream* output) { + { + io::OstreamOutputStream zero_copy_output(output); + if (!SerializeDelimitedToZeroCopyStream(message, &zero_copy_output)) return false; + } + return output->good(); +} + +bool ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof) { + google::protobuf::io::CodedInputStream coded_input(input); + return ParseDelimitedFromCodedStream(message, &coded_input, clean_eof); +} + +bool ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof) { + if (clean_eof != NULL) *clean_eof = false; + int start = input->CurrentPosition(); + + // Read the size. + uint32 size; + if (!input->ReadVarint32(&size)) { + if (clean_eof != NULL) *clean_eof = input->CurrentPosition() == start; + return false; + } + + // Tell the stream not to read beyond that size. + google::protobuf::io::CodedInputStream::Limit limit = input->PushLimit(size); + + // Parse the message. + if (!message->MergeFromCodedStream(input)) return false; + if (!input->ConsumedEntireMessage()) return false; + + // Release the limit. + input->PopLimit(limit); + + return true; +} + +bool SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output) { + google::protobuf::io::CodedOutputStream coded_output(output); + return SerializeDelimitedToCodedStream(message, &coded_output); +} + +bool SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output) { + // Write the size. + int size = message.ByteSize(); + output->WriteVarint32(size); + + // Write the content. + uint8* buffer = output->GetDirectBufferForNBytesAndAdvance(size); + if (buffer != NULL) { + // Optimization: The message fits in one buffer, so use the faster + // direct-to-array serialization path. + message.SerializeWithCachedSizesToArray(buffer); + } else { + // Slightly-slower path when the message is multiple buffers. + message.SerializeWithCachedSizes(output); + if (output->HadError()) return false; + } + + return true; +} + +} // namespace util +} // namespace protobuf +} // namespace google diff --git a/src/google/protobuf/util/delimited_message_util.h b/src/google/protobuf/util/delimited_message_util.h new file mode 100644 index 0000000000..302d4781b3 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util.h @@ -0,0 +1,66 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#ifndef GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ +#define GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ + +#include + +#include +#include +#include + +namespace google { +namespace protobuf { +namespace util { + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +// +// Note that if you want to *read* a delimited message from a file descriptor +// or istream, you will need to construct an io::FileInputStream or +// io::OstreamInputStream (implementations of io::ZeroCopyStream) and use the +// utility function ParseDelimitedFromZeroCopyStream(). You must then +// continue to use the same ZeroCopyInputStream to read all further data from +// the stream until EOF. This is because these ZeroCopyInputStream +// implementations are buffered: they read a big chunk of data at a time, +// then parse it. As a result, they may read past the end of the delimited +// message. There is no way for them to push the extra data back into the +// underlying source, so instead you must keep using the same stream object. +bool LIBPROTOBUF_EXPORT SerializeDelimitedToFileDescriptor(const MessageLite& message, int file_descriptor); + +bool LIBPROTOBUF_EXPORT SerializeDelimitedToOstream(const MessageLite& message, ostream* output); + +// Read a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally parsing consumes the entire input. A delimited message +// is a varint encoding the message size followed by a message of exactly +// that size. +// +// If |clean_eof| is not NULL, then it will be set to indicate whether the +// stream ended cleanly. That is, if the stream ends without this method +// having read any data at all from it, then *clean_eof will be set true, +// otherwise it will be set false. Note that these methods return false +// on EOF, but they also return false on other errors, so |clean_eof| is +// needed to distinguish a clean end from errors. +bool LIBPROTOBUF_EXPORT ParseDelimitedFromZeroCopyStream(MessageLite* message, io::ZeroCopyInputStream* input, bool* clean_eof); + +bool LIBPROTOBUF_EXPORT ParseDelimitedFromCodedStream(MessageLite* message, io::CodedInputStream* input, bool* clean_eof); + +// Write a single size-delimited message from the given stream. Delimited +// format allows a single file or stream to contain multiple messages, +// whereas normally writing multiple non-delimited messages to the same +// stream would cause them to be merged. A delimited message is a varint +// encoding the message size followed by a message of exactly that size. +bool LIBPROTOBUF_EXPORT SerializeDelimitedToZeroCopyStream(const MessageLite& message, io::ZeroCopyOutputStream* output); + +bool LIBPROTOBUF_EXPORT SerializeDelimitedToCodedStream(const MessageLite& message, io::CodedOutputStream* output); + +} // namespace util +} // namespace protobuf +} // namespace google + +#endif // GOOGLE_PROTOBUF_UTIL_DELIMITED_MESSAGE_UTIL_H__ diff --git a/src/google/protobuf/util/delimited_message_util_test.cc b/src/google/protobuf/util/delimited_message_util_test.cc new file mode 100644 index 0000000000..b261d43cc9 --- /dev/null +++ b/src/google/protobuf/util/delimited_message_util_test.cc @@ -0,0 +1,57 @@ +// Adapted from the patch of kenton@google.com (Kenton Varda) +// See https://github.com/google/protobuf/pull/710 for details. + +#include + +#include + +#include +#include +#include +#include + +namespace google { +namespace protobuf { +namespace util { + +TEST(DelimitedMessageUtilTest, DelimitedMessages) { + stringstream stream; + + { + protobuf_unittest::TestAllTypes message1; + TestUtil::SetAllFields(&message1); + EXPECT_TRUE(SerializeDelimitedToOstream(message1, &stream)); + + protobuf_unittest::TestPackedTypes message2; + TestUtil::SetPackedFields(&message2); + EXPECT_TRUE(SerializeDelimitedToOstream(message2, &stream)); + } + + { + bool clean_eof; + io::IstreamInputStream zstream(&stream); + + protobuf_unittest::TestAllTypes message1; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message1, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectAllFieldsSet(message1); + + protobuf_unittest::TestPackedTypes message2; + clean_eof = true; + EXPECT_TRUE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_FALSE(clean_eof); + TestUtil::ExpectPackedFieldsSet(message2); + + clean_eof = false; + EXPECT_FALSE(ParseDelimitedFromZeroCopyStream(&message2, + &zstream, &clean_eof)); + EXPECT_TRUE(clean_eof); + } +} + +} // namespace util +} // namespace protobuf +} // namespace google