From bc618eed71f1c2a2b69351a6aa60e8bb460773b9 Mon Sep 17 00:00:00 2001 From: chedeti Date: Sun, 31 Jul 2016 15:35:51 -0700 Subject: [PATCH] thrift serializer --- Makefile | 3 + build.yaml | 9 + .../grpc++/impl/codegen/thrift_serializer.h | 148 +++++++++++++++ .../impl/codegen/thrift_serializer_inl.h | 169 ++++++++++++++++++ include/grpc++/impl/codegen/thrift_utils.h | 90 ++++++++++ tools/run_tests/sources_and_headers.json | 22 ++- .../grpc++_test_util/grpc++_test_util.vcxproj | 3 + .../grpc++_test_util.vcxproj.filters | 9 + 8 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 include/grpc++/impl/codegen/thrift_serializer.h create mode 100644 include/grpc++/impl/codegen/thrift_serializer_inl.h create mode 100644 include/grpc++/impl/codegen/thrift_utils.h diff --git a/Makefile b/Makefile index 8b6114bd7f9..986c25780f2 100644 --- a/Makefile +++ b/Makefile @@ -3923,6 +3923,9 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/time.h \ include/grpc++/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ + include/grpc++/impl/codegen/thrift_serializer.h \ + include/grpc++/impl/codegen/thrift_serializer_inl.h \ + include/grpc++/impl/codegen/thrift_utils.h \ LIBGRPC++_TEST_UTIL_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(LIBGRPC++_TEST_UTIL_SRC)))) diff --git a/build.yaml b/build.yaml index 37c95ccdc0a..35af7a6c7a5 100644 --- a/build.yaml +++ b/build.yaml @@ -780,6 +780,14 @@ filegroups: - src/cpp/ext/reflection.pb.cc uses: - grpc++_codegen_proto +- name: thrift_util + language: c++ + public_headers: + - include/grpc++/impl/codegen/thrift_serializer.h + - include/grpc++/impl/codegen/thrift_serializer_inl.h + - include/grpc++/impl/codegen/thrift_utils.h + uses: + - grpc++_codegen_base libs: - name: gpr build: all @@ -1021,6 +1029,7 @@ libs: - grpc++_codegen_base_src - grpc++_codegen_proto - grpc++_config_proto + - thrift_util - name: grpc++_unsecure build: all language: c++ diff --git a/include/grpc++/impl/codegen/thrift_serializer.h b/include/grpc++/impl/codegen/thrift_serializer.h new file mode 100644 index 00000000000..315d76cf2a5 --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_serializer.h @@ -0,0 +1,148 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H + #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_H + +#include +#include + +#include +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace util { + +using apache::thrift::protocol::TBinaryProtocolT; +using apache::thrift::protocol::TCompactProtocolT; +using apache::thrift::protocol::TNetworkBigEndian; +using apache::thrift::transport::TMemoryBuffer; +using apache::thrift::transport::TBufferBase; +using apache::thrift::transport::TTransport; +using std::shared_ptr; + + +template +class ThriftSerializer { +public: + ThriftSerializer() + : prepared_ (false) + , lastDeserialized_ (false) + , serializeVersion_ (false) {} + + /** + * Serialize the passed type into the internal buffer + * and returns a pointer to internal buffer and its size + * + */ + template + void serialize(const T& fields, const uint8_t** serializedBuffer, + size_t* serializedLen); + + /** + * Serialize the passed type into the byte buffer + */ + template + void serialize(const T& fields, grpc_byte_buffer** bp); + + /** + * Deserialize the passed char array into the passed type, returns the number + * of bytes that have been consumed from the passed string. + */ + template + uint32_t deserialize(const uint8_t* serializedBuffer, size_t length, + T* fields); + + /** + * Deserialize the passed byte buffer to passed type, returns the number + * of bytes consumed from byte buffer + */ + template + uint32_t deserialize(grpc_byte_buffer* buffer, T* msg); + + void setSerializeVersion(bool value); + + virtual ~ThriftSerializer() {} + + + /** + * Set the container size limit to deserialize + * This function should be called after buffer_ is initialized + */ + void setContainerSizeLimit(int32_t container_limit) { + if (!prepared_) { + prepare(); + } + protocol_->setContainerSizeLimit(container_limit); + } + + /** + * Set the string size limit to deserialize + * This function should be called after buffer_ is initialized + */ + void setStringSizeLimit(int32_t string_limit) { + if (!prepared_) { + prepare(); + } + protocol_->setStringSizeLimit(string_limit); + } + + + private: + void prepare(); + + private: + typedef P Protocol; + bool prepared_; + bool lastDeserialized_; + boost::shared_ptr buffer_; + shared_ptr protocol_; + bool serializeVersion_; +}; // ThriftSerializer + +template +struct ThriftSerializerBinary : public ThriftSerializer > {}; + + +template +struct ThriftSerializerCompact : public ThriftSerializer >{ }; + +}}} // namespace apache::thrift::util + +#include + +#endif diff --git a/include/grpc++/impl/codegen/thrift_serializer_inl.h b/include/grpc++/impl/codegen/thrift_serializer_inl.h new file mode 100644 index 00000000000..866ecf6312d --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_serializer_inl.h @@ -0,0 +1,169 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + + #ifndef GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H + #define GRPCXX_IMPL_CODEGEN_THRIFT_SERIALIZER_INL_H + +#include +#include +#include +#include +#include +#include +#include + +namespace apache { +namespace thrift { +namespace util { + +using apache::thrift::protocol::TMessageType; + +template +template +void ThriftSerializer::serialize(const T& fields, + const uint8_t** serializedBuffer, size_t* serializedLen) { + + // prepare or reset buffer + if (!prepared_ || lastDeserialized_) { + prepare(); + } else { + buffer_->resetBuffer(); + } + lastDeserialized_ = false; + + // if required serialize protocol version + if (serializeVersion_) { + protocol_->writeMessageBegin("", TMessageType(0), 0); + } + + // serilaize fields into buffer + fields.write(protocol_.get()); + + // write the end of message + if (serializeVersion_) { + protocol_->writeMessageEnd(); + } + + // assign buffer to string + uint8_t* byteBuffer; + uint32_t byteBufferSize; + buffer_->getBuffer(&byteBuffer, &byteBufferSize); + *serializedBuffer = byteBuffer; + *serializedLen = byteBufferSize; +} + +template +template +void ThriftSerializer::serialize(const T& fields, grpc_byte_buffer** bp) { + + const uint8_t* byteBuffer; + size_t byteBufferSize; + serialize(fields, &byteBuffer, &byteBufferSize); + + gpr_slice slice = gpr_slice_from_copied_buffer((char*)byteBuffer,byteBufferSize); + + *bp = grpc_raw_byte_buffer_create(&slice, 1); + + gpr_slice_unref(slice); +} + +template +template +uint32_t ThriftSerializer::deserialize(const uint8_t* serializedBuffer, + size_t length, T* fields) { + // prepare buffer if necessary + if (!prepared_) { + prepare(); + } + lastDeserialized_ = true; + + //reset buffer transport + buffer_->resetBuffer((uint8_t*)serializedBuffer, length); + + // read the protocol version if necessary + if (serializeVersion_) { + std::string name = ""; + TMessageType mt = (TMessageType) 0; + int32_t seq_id = 0; + protocol_->readMessageBegin(name, mt, seq_id); + } + + // deserialize buffer into fields + uint32_t len = fields->read(protocol_.get()); + + // read the end of message + if (serializeVersion_) { + protocol_->readMessageEnd(); + } + + return len; +} + +template +template +uint32_t ThriftSerializer::deserialize(grpc_byte_buffer* bp, T* fields) { + grpc_byte_buffer_reader reader; + grpc_byte_buffer_reader_init(&reader, bp); + + gpr_slice slice = grpc_byte_buffer_reader_readall(&reader); + + uint32_t len = deserialize(GPR_SLICE_START_PTR(slice), GPR_SLICE_LENGTH(slice), fields); + + gpr_slice_unref(slice); + + grpc_byte_buffer_reader_destroy(&reader); + + return len; +} + +template +void ThriftSerializer::setSerializeVersion(bool value) { + serializeVersion_ = value; +} + +template +void +ThriftSerializer::prepare() +{ + + buffer_.reset(new TMemoryBuffer()); + + // create a protocol for the memory buffer transport + protocol_.reset(new Protocol(buffer_)); + + prepared_ = true; +} + +}}} // namespace apache::thrift::util + +#endif diff --git a/include/grpc++/impl/codegen/thrift_utils.h b/include/grpc++/impl/codegen/thrift_utils.h new file mode 100644 index 00000000000..629441149fd --- /dev/null +++ b/include/grpc++/impl/codegen/thrift_utils.h @@ -0,0 +1,90 @@ +/* + * + * Copyright 2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H +#define GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace grpc { + +using apache::thrift::util::ThriftSerializerCompact; + +template +class SerializationTraits::value>::type> { + public: + + static Status Serialize(const T& msg, + grpc_byte_buffer** bp, bool* own_buffer) { + + *own_buffer = true; + + ThriftSerializerCompact serializer; + + serializer.serialize(msg, bp); + + return Status(StatusCode::OK, "ok"); + } + + static Status Deserialize(grpc_byte_buffer* buffer, + T* msg, + int max_message_size) { + if (!buffer) { + return Status(StatusCode::INTERNAL, "No payload"); + } + + ThriftSerializerCompact deserializer; + deserializer.deserialize(buffer, msg); + + grpc_byte_buffer_destroy(buffer); + + return Status(StatusCode::OK, "ok"); + } +}; + +} // namespace grpc + +#endif // GRPCXX_IMPL_CODEGEN_THRIFT_UTILS_H diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index e85dd2c8200..06ac86fc9c4 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -4411,7 +4411,8 @@ "grpc++_codegen_base_src", "grpc++_codegen_proto", "grpc++_config_proto", - "grpc_test_util" + "grpc_test_util", + "thrift_util" ], "headers": [ "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h", @@ -6784,5 +6785,24 @@ ], "third_party": false, "type": "filegroup" + }, + { + "deps": [ + "grpc++_codegen_base" + ], + "headers": [ + "include/grpc++/impl/codegen/thrift_serializer.h", + "include/grpc++/impl/codegen/thrift_serializer_inl.h", + "include/grpc++/impl/codegen/thrift_utils.h" + ], + "language": "c++", + "name": "thrift_util", + "src": [ + "include/grpc++/impl/codegen/thrift_serializer.h", + "include/grpc++/impl/codegen/thrift_serializer_inl.h", + "include/grpc++/impl/codegen/thrift_utils.h" + ], + "third_party": false, + "type": "filegroup" } ] diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj index d0fca9ba65a..96ce3b89164 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj @@ -200,6 +200,9 @@ + + + diff --git a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters index cc98c604080..2105e672df9 100644 --- a/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc++_test_util/grpc++_test_util.vcxproj.filters @@ -192,6 +192,15 @@ include\grpc++\impl\codegen + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen + + + include\grpc++\impl\codegen +