From c1f837ce796b6cd8e46da9a97df2e3f8f267639f Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 4 May 2017 18:17:13 -0700 Subject: [PATCH 01/10] Make stream compression module --- CMakeLists.txt | 36 +++ Makefile | 40 +++ binding.gyp | 1 + build.yaml | 12 + config.m4 | 1 + config.w32 | 1 + gRPC-Core.podspec | 3 + grpc.gemspec | 2 + package.xml | 2 + .../chttp2/transport/stream_compression.c | 206 ++++++++++++ .../chttp2/transport/stream_compression.h | 97 ++++++ src/python/grpcio/grpc_core_dependencies.py | 1 + .../compression/stream_compression_test.c | 303 ++++++++++++++++++ tools/doxygen/Doxyfile.core.internal | 2 + .../generated/sources_and_headers.json | 20 ++ tools/run_tests/generated/tests.json | 22 ++ vsprojects/buildtests_c.sln | 27 ++ vsprojects/vcxproj/grpc/grpc.vcxproj | 3 + vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 + .../grpc_unsecure/grpc_unsecure.vcxproj | 3 + .../grpc_unsecure.vcxproj.filters | 6 + .../stream_compression_test.vcxproj | 199 ++++++++++++ .../stream_compression_test.vcxproj.filters | 21 ++ 23 files changed, 1014 insertions(+) create mode 100644 src/core/ext/transport/chttp2/transport/stream_compression.c create mode 100644 src/core/ext/transport/chttp2/transport/stream_compression.h create mode 100644 test/core/compression/stream_compression_test.c create mode 100644 vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj create mode 100644 vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj.filters diff --git a/CMakeLists.txt b/CMakeLists.txt index 852eb2bf6c8..e9aae4f06bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -510,6 +510,7 @@ if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c socket_utils_test) endif() add_dependencies(buildtests_c status_conversion_test) +add_dependencies(buildtests_c stream_compression_test) add_dependencies(buildtests_c stream_owned_slice_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_c tcp_client_posix_test) @@ -1067,6 +1068,7 @@ add_library(grpc src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c + src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -1409,6 +1411,7 @@ add_library(grpc_cronet src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c + src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -1988,6 +1991,7 @@ add_library(grpc_unsecure src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c + src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -2496,6 +2500,7 @@ add_library(grpc++_cronet src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c + src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -8125,6 +8130,37 @@ target_link_libraries(status_conversion_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) +add_executable(stream_compression_test + test/core/compression/stream_compression_test.c +) + + +target_include_directories(stream_compression_test + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include + PRIVATE ${BORINGSSL_ROOT_DIR}/include + PRIVATE ${PROTOBUF_ROOT_DIR}/src + PRIVATE ${BENCHMARK_ROOT_DIR}/include + PRIVATE ${ZLIB_ROOT_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/zlib + PRIVATE ${CARES_BUILD_INCLUDE_DIR} + PRIVATE ${CARES_INCLUDE_DIR} + PRIVATE ${CARES_PLATFORM_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/cares/cares + PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/third_party/gflags/include +) + +target_link_libraries(stream_compression_test + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_test_util + grpc + gpr_test_util + gpr +) + +endif (gRPC_BUILD_TESTS) +if (gRPC_BUILD_TESTS) + add_executable(stream_owned_slice_test test/core/transport/stream_owned_slice_test.c ) diff --git a/Makefile b/Makefile index 0ca788592c4..db9e050f87b 100644 --- a/Makefile +++ b/Makefile @@ -1074,6 +1074,7 @@ sockaddr_utils_test: $(BINDIR)/$(CONFIG)/sockaddr_utils_test socket_utils_test: $(BINDIR)/$(CONFIG)/socket_utils_test ssl_server_fuzzer: $(BINDIR)/$(CONFIG)/ssl_server_fuzzer status_conversion_test: $(BINDIR)/$(CONFIG)/status_conversion_test +stream_compression_test: $(BINDIR)/$(CONFIG)/stream_compression_test stream_owned_slice_test: $(BINDIR)/$(CONFIG)/stream_owned_slice_test tcp_client_posix_test: $(BINDIR)/$(CONFIG)/tcp_client_posix_test tcp_client_uv_test: $(BINDIR)/$(CONFIG)/tcp_client_uv_test @@ -1441,6 +1442,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/sockaddr_utils_test \ $(BINDIR)/$(CONFIG)/socket_utils_test \ $(BINDIR)/$(CONFIG)/status_conversion_test \ + $(BINDIR)/$(CONFIG)/stream_compression_test \ $(BINDIR)/$(CONFIG)/stream_owned_slice_test \ $(BINDIR)/$(CONFIG)/tcp_client_posix_test \ $(BINDIR)/$(CONFIG)/tcp_client_uv_test \ @@ -1921,6 +1923,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/socket_utils_test || ( echo test socket_utils_test failed ; exit 1 ) $(E) "[RUN] Testing status_conversion_test" $(Q) $(BINDIR)/$(CONFIG)/status_conversion_test || ( echo test status_conversion_test failed ; exit 1 ) + $(E) "[RUN] Testing stream_compression_test" + $(Q) $(BINDIR)/$(CONFIG)/stream_compression_test || ( echo test stream_compression_test failed ; exit 1 ) $(E) "[RUN] Testing stream_owned_slice_test" $(Q) $(BINDIR)/$(CONFIG)/stream_owned_slice_test || ( echo test stream_owned_slice_test failed ; exit 1 ) $(E) "[RUN] Testing tcp_client_posix_test" @@ -3016,6 +3020,7 @@ LIBGRPC_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ + src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -3356,6 +3361,7 @@ LIBGRPC_CRONET_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ + src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -3904,6 +3910,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ + src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -4396,6 +4403,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ + src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -12218,6 +12226,38 @@ endif endif +STREAM_COMPRESSION_TEST_SRC = \ + test/core/compression/stream_compression_test.c \ + +STREAM_COMPRESSION_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(STREAM_COMPRESSION_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/stream_compression_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/stream_compression_test: $(STREAM_COMPRESSION_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(STREAM_COMPRESSION_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/stream_compression_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/compression/stream_compression_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_stream_compression_test: $(STREAM_COMPRESSION_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(STREAM_COMPRESSION_TEST_OBJS:.o=.dep) +endif +endif + + STREAM_OWNED_SLICE_TEST_SRC = \ test/core/transport/stream_owned_slice_test.c \ diff --git a/binding.gyp b/binding.gyp index d85cf0be14f..11c06941f70 100644 --- a/binding.gyp +++ b/binding.gyp @@ -791,6 +791,7 @@ 'src/core/ext/transport/chttp2/transport/huffsyms.c', 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', 'src/core/ext/transport/chttp2/transport/parsing.c', + 'src/core/ext/transport/chttp2/transport/stream_compression.c', 'src/core/ext/transport/chttp2/transport/stream_lists.c', 'src/core/ext/transport/chttp2/transport/stream_map.c', 'src/core/ext/transport/chttp2/transport/varint.c', diff --git a/build.yaml b/build.yaml index 940ca8902dc..ee71e3fa8ae 100644 --- a/build.yaml +++ b/build.yaml @@ -738,6 +738,7 @@ filegroups: - src/core/ext/transport/chttp2/transport/huffsyms.h - src/core/ext/transport/chttp2/transport/incoming_metadata.h - src/core/ext/transport/chttp2/transport/internal.h + - src/core/ext/transport/chttp2/transport/stream_compression.h - src/core/ext/transport/chttp2/transport/stream_map.h - src/core/ext/transport/chttp2/transport/varint.h src: @@ -758,6 +759,7 @@ filegroups: - src/core/ext/transport/chttp2/transport/huffsyms.c - src/core/ext/transport/chttp2/transport/incoming_metadata.c - src/core/ext/transport/chttp2/transport/parsing.c + - src/core/ext/transport/chttp2/transport/stream_compression.c - src/core/ext/transport/chttp2/transport/stream_lists.c - src/core/ext/transport/chttp2/transport/stream_map.c - src/core/ext/transport/chttp2/transport/varint.c @@ -2987,6 +2989,16 @@ targets: - grpc - gpr_test_util - gpr +- name: stream_compression_test + build: test + language: c + src: + - test/core/compression/stream_compression_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: stream_owned_slice_test build: test language: c diff --git a/config.m4 b/config.m4 index eb0df98acdf..6bf72eea8ad 100644 --- a/config.m4 +++ b/config.m4 @@ -228,6 +228,7 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ + src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ diff --git a/config.w32 b/config.w32 index cca36040455..46779e0723d 100644 --- a/config.w32 +++ b/config.w32 @@ -205,6 +205,7 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\transport\\chttp2\\transport\\huffsyms.c " + "src\\core\\ext\\transport\\chttp2\\transport\\incoming_metadata.c " + "src\\core\\ext\\transport\\chttp2\\transport\\parsing.c " + + "src\\core\\ext\\transport\\chttp2\\transport\\stream_compression.c " + "src\\core\\ext\\transport\\chttp2\\transport\\stream_lists.c " + "src\\core\\ext\\transport\\chttp2\\transport\\stream_map.c " + "src\\core\\ext\\transport\\chttp2\\transport\\varint.c " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 782ebb8ae13..0062bf9f145 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -370,6 +370,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/huffsyms.h', 'src/core/ext/transport/chttp2/transport/incoming_metadata.h', 'src/core/ext/transport/chttp2/transport/internal.h', + 'src/core/ext/transport/chttp2/transport/stream_compression.h', 'src/core/ext/transport/chttp2/transport/stream_map.h', 'src/core/ext/transport/chttp2/transport/varint.h', 'src/core/ext/transport/chttp2/alpn/alpn.h', @@ -603,6 +604,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/huffsyms.c', 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', 'src/core/ext/transport/chttp2/transport/parsing.c', + 'src/core/ext/transport/chttp2/transport/stream_compression.c', 'src/core/ext/transport/chttp2/transport/stream_lists.c', 'src/core/ext/transport/chttp2/transport/stream_map.c', 'src/core/ext/transport/chttp2/transport/varint.c', @@ -853,6 +855,7 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/huffsyms.h', 'src/core/ext/transport/chttp2/transport/incoming_metadata.h', 'src/core/ext/transport/chttp2/transport/internal.h', + 'src/core/ext/transport/chttp2/transport/stream_compression.h', 'src/core/ext/transport/chttp2/transport/stream_map.h', 'src/core/ext/transport/chttp2/transport/varint.h', 'src/core/ext/transport/chttp2/alpn/alpn.h', diff --git a/grpc.gemspec b/grpc.gemspec index 18ef37a0774..daf3c99702c 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -301,6 +301,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/transport/huffsyms.h ) s.files += %w( src/core/ext/transport/chttp2/transport/incoming_metadata.h ) s.files += %w( src/core/ext/transport/chttp2/transport/internal.h ) + s.files += %w( src/core/ext/transport/chttp2/transport/stream_compression.h ) s.files += %w( src/core/ext/transport/chttp2/transport/stream_map.h ) s.files += %w( src/core/ext/transport/chttp2/transport/varint.h ) s.files += %w( src/core/ext/transport/chttp2/alpn/alpn.h ) @@ -534,6 +535,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/transport/huffsyms.c ) s.files += %w( src/core/ext/transport/chttp2/transport/incoming_metadata.c ) s.files += %w( src/core/ext/transport/chttp2/transport/parsing.c ) + s.files += %w( src/core/ext/transport/chttp2/transport/stream_compression.c ) s.files += %w( src/core/ext/transport/chttp2/transport/stream_lists.c ) s.files += %w( src/core/ext/transport/chttp2/transport/stream_map.c ) s.files += %w( src/core/ext/transport/chttp2/transport/varint.c ) diff --git a/package.xml b/package.xml index 10cb2e63ff1..1e8f408c218 100644 --- a/package.xml +++ b/package.xml @@ -315,6 +315,7 @@ + @@ -548,6 +549,7 @@ + diff --git a/src/core/ext/transport/chttp2/transport/stream_compression.c b/src/core/ext/transport/chttp2/transport/stream_compression.c new file mode 100644 index 00000000000..b2869102372 --- /dev/null +++ b/src/core/ext/transport/chttp2/transport/stream_compression.c @@ -0,0 +1,206 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include +#include + +#include "src/core/ext/transport/chttp2/transport/stream_compression.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/slice/slice_internal.h" + +#define OUTPUT_BLOCK_SIZE (1024) + +static bool gzip_flate(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, int flush, + bool *end_of_context) { + GPR_ASSERT(flush == 0 || flush == Z_SYNC_FLUSH || flush == Z_FINISH); + /* Full flush is not allowed when inflating. */ + GPR_ASSERT(!(ctx->flate == inflate && (flush == Z_FINISH))); + + grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT; + int r; + bool eoc = false; + size_t original_max_output_size = max_output_size; + while (max_output_size > 0 && (in->length > 0 || flush) && !eoc) { + size_t slice_size = max_output_size < OUTPUT_BLOCK_SIZE ? max_output_size + : OUTPUT_BLOCK_SIZE; + grpc_slice slice_out = GRPC_SLICE_MALLOC(slice_size); + ctx->zs.avail_out = (uInt)slice_size; + ctx->zs.next_out = GRPC_SLICE_START_PTR(slice_out); + while (ctx->zs.avail_out > 0 && in->length > 0 && !eoc) { + grpc_slice slice = grpc_slice_buffer_take_first(in); + ctx->zs.avail_in = (uInt)GRPC_SLICE_LENGTH(slice); + ctx->zs.next_in = GRPC_SLICE_START_PTR(slice); + r = ctx->flate(&ctx->zs, Z_NO_FLUSH); + if (r < 0 && r != Z_BUF_ERROR) { + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } else if (r == Z_STREAM_END && ctx->flate == inflate) { + eoc = true; + } + if (ctx->zs.avail_in > 0) { + grpc_slice_buffer_undo_take_first( + in, + grpc_slice_sub(slice, GRPC_SLICE_LENGTH(slice) - ctx->zs.avail_in, + GRPC_SLICE_LENGTH(slice))); + } + grpc_slice_unref_internal(&exec_ctx, slice); + } + if (flush != 0 && ctx->zs.avail_out > 0 && !eoc) { + GPR_ASSERT(in->length == 0); + r = ctx->flate(&ctx->zs, flush); + if (flush == Z_SYNC_FLUSH) { + switch (r) { + case Z_OK: + /* Maybe flush is not complete; just made some partial progress. */ + if (ctx->zs.avail_out > 0) { + flush = 0; + } + break; + case Z_BUF_ERROR: + case Z_STREAM_END: + flush = 0; + break; + default: + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } + } else if (flush == Z_FINISH) { + switch (r) { + case Z_OK: + case Z_BUF_ERROR: + /* Wait for the next loop to assign additional output space. */ + GPR_ASSERT(ctx->zs.avail_out == 0); + break; + case Z_STREAM_END: + flush = 0; + break; + default: + gpr_log(GPR_ERROR, "zlib error (%d)", r); + grpc_slice_unref_internal(&exec_ctx, slice_out); + grpc_exec_ctx_finish(&exec_ctx); + return false; + } + } + } + + if (ctx->zs.avail_out == 0) { + grpc_slice_buffer_add(out, slice_out); + } else if (ctx->zs.avail_out < slice_size) { + slice_out.data.refcounted.length -= ctx->zs.avail_out; + grpc_slice_buffer_add(out, slice_out); + } else { + grpc_slice_unref_internal(&exec_ctx, slice_out); + } + max_output_size -= (slice_size - ctx->zs.avail_out); + } + grpc_exec_ctx_finish(&exec_ctx); + if (end_of_context) { + *end_of_context = eoc; + } + if (output_size) { + *output_size = original_max_output_size - max_output_size; + } + return true; +} + +bool grpc_stream_compress(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush) { + GPR_ASSERT(ctx->flate == deflate); + int gzip_flush; + switch (flush) { + case GRPC_STREAM_COMPRESSION_FLUSH_NONE: + gzip_flush = 0; + break; + case GRPC_STREAM_COMPRESSION_FLUSH_SYNC: + gzip_flush = Z_SYNC_FLUSH; + break; + case GRPC_STREAM_COMPRESSION_FLUSH_FINISH: + gzip_flush = Z_FINISH; + break; + default: + gzip_flush = 0; + } + return gzip_flate(ctx, in, out, output_size, max_output_size, gzip_flush, + NULL); +} + +bool grpc_stream_decompress(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context) { + GPR_ASSERT(ctx->flate == inflate); + return gzip_flate(ctx, in, out, output_size, max_output_size, Z_SYNC_FLUSH, + end_of_context); +} + +grpc_stream_compression_context *grpc_stream_compression_context_create( + grpc_stream_compression_method method) { + grpc_stream_compression_context *ctx = + gpr_zalloc(sizeof(grpc_stream_compression_context)); + int r; + if (ctx == NULL) { + return NULL; + } + if (method == GRPC_STREAM_COMPRESSION_DECOMPRESS) { + r = inflateInit2(&ctx->zs, 0x1F); + ctx->flate = inflate; + } else { + r = deflateInit2(&ctx->zs, Z_DEFAULT_COMPRESSION, Z_DEFLATED, 0x1F, 8, + Z_DEFAULT_STRATEGY); + ctx->flate = deflate; + } + if (r != Z_OK) { + gpr_free(ctx); + return NULL; + } + + return ctx; +} + +void grpc_stream_compression_context_destroy( + grpc_stream_compression_context *ctx) { + if (ctx->flate == inflate) { + inflateEnd(&ctx->zs); + } else { + deflateEnd(&ctx->zs); + } + gpr_free(ctx); +} diff --git a/src/core/ext/transport/chttp2/transport/stream_compression.h b/src/core/ext/transport/chttp2/transport/stream_compression.h new file mode 100644 index 00000000000..21ae2fd32d1 --- /dev/null +++ b/src/core/ext/transport/chttp2/transport/stream_compression.h @@ -0,0 +1,97 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRONSPORT_STREAM_COMPRESSION_H +#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRONSPORT_STREAM_COMPRESSION_H + +#include + +#include +#include + +/* Stream compression/decompression context */ +typedef struct grpc_stream_compression_context { + z_stream zs; + int (*flate)(z_stream *zs, int flush); +} grpc_stream_compression_context; + +typedef enum grpc_stream_compression_method { + GRPC_STREAM_COMPRESSION_COMPRESS, + GRPC_STREAM_COMPRESSION_DECOMPRESS +} grpc_stream_compression_method; + +typedef enum grpc_stream_compression_flush { + GRPC_STREAM_COMPRESSION_FLUSH_NONE, + GRPC_STREAM_COMPRESSION_FLUSH_SYNC, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH +} grpc_stream_compression_flush; + +/** + * Compress bytes provided in \a in with a given context, with an optional flush + * at the end of compression. Emits at most \a max_output_size compressed bytes + * into \a out. If all the bytes in input buffer \a in are depleted and \a flush + * is not GRPC_STREAM_COMPRESSION_FLUSH_NONE, the corresponding flush method is + * executed. The total number of bytes emitted is outputed in \a output_size. + */ +bool grpc_stream_compress(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + grpc_stream_compression_flush flush); + +/** + * Decompress bytes provided in \a in with a given context. Emits at most \a + * max_output_size decompressed bytes into \a out. If decompression process + * reached the end of a gzip stream, \a end_of_context is set to true; otherwise + * it is set to false. The total number of bytes emitted is outputed in \a + * output_size. + */ +bool grpc_stream_decompress(grpc_stream_compression_context *ctx, + grpc_slice_buffer *in, grpc_slice_buffer *out, + size_t *output_size, size_t max_output_size, + bool *end_of_context); + +/** + * Creates a stream compression context. \a pending_bytes_buffer is the input + * buffer for compression/decompression operations. \a method specifies whether + * the context is for compression or decompression. + */ +grpc_stream_compression_context *grpc_stream_compression_context_create( + grpc_stream_compression_method method); + +/** + * Destroys a stream compression context. + */ +void grpc_stream_compression_context_destroy( + grpc_stream_compression_context *ctx); + +#endif diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 5819a624f7c..aa7e1b53178 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -204,6 +204,7 @@ CORE_SOURCE_FILES = [ 'src/core/ext/transport/chttp2/transport/huffsyms.c', 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', 'src/core/ext/transport/chttp2/transport/parsing.c', + 'src/core/ext/transport/chttp2/transport/stream_compression.c', 'src/core/ext/transport/chttp2/transport/stream_lists.c', 'src/core/ext/transport/chttp2/transport/stream_map.c', 'src/core/ext/transport/chttp2/transport/varint.c', diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c new file mode 100644 index 00000000000..28700c8f703 --- /dev/null +++ b/test/core/compression/stream_compression_test.c @@ -0,0 +1,303 @@ +/* + * + * Copyright 2017, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include + +#include +#include +#include +#include + +#include "src/core/ext/transport/chttp2/transport/stream_compression.h" + +static void generate_random_payload(char *payload, size_t size) { + size_t i; + static const char chars[] = "abcdefghijklmnopqrstuvwxyz1234567890"; + for (i = 0; i < size - 1; ++i) { + payload[i] = chars[rand() % (int)(sizeof(chars) - 1)]; + } + payload[size - 1] = '\0'; +} + +static bool slice_buffer_equals_string(grpc_slice_buffer *buf, const char *str) { + size_t i; + if (buf->length != strlen(str)) { + return false; + } + size_t pointer = 0; + for (i = 0; i < buf->count; i++) { + size_t slice_len = GRPC_SLICE_LENGTH(buf->slices[i]); + if (0 != strncmp(str + pointer, (char *)GRPC_SLICE_START_PTR(buf->slices[i]), slice_len)) { + return false; + } + pointer += slice_len; + } + return true; +} + +static void test_stream_compression_simple_compress_decompress() { + const char test_str[] = "aaaaaaabbbbbbbccccccctesttesttest"; + grpc_slice_buffer source, relay, sink; + grpc_slice_buffer_init(&source); + grpc_slice_buffer_init(&relay); + grpc_slice_buffer_init(&sink); + grpc_stream_compression_context *compress_ctx = + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context *decompress_ctx = + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_DECOMPRESS); + grpc_slice slice = grpc_slice_from_static_string(test_str); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + bool end_of_context; + size_t output_size; + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + ~(size_t)0, &end_of_context)); + GPR_ASSERT(output_size = sizeof(test_str) - 1); + grpc_stream_compression_context_destroy(compress_ctx); + grpc_stream_compression_context_destroy(decompress_ctx); + + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str)); + + grpc_slice_buffer_destroy(&source); + grpc_slice_buffer_destroy(&relay); + grpc_slice_buffer_destroy(&sink); +} + +static void +test_stream_compression_simple_compress_decompress_with_output_size_constraint() { + const char test_str[] = "aaaaaaabbbbbbbccccccctesttesttest"; + grpc_slice_buffer source, relay, sink; + grpc_slice_buffer_init(&source); + grpc_slice_buffer_init(&relay); + grpc_slice_buffer_init(&sink); + grpc_stream_compression_context *compress_ctx = + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context *decompress_ctx = + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_DECOMPRESS); + grpc_slice slice = grpc_slice_from_static_string(test_str); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + grpc_stream_compression_context_destroy(compress_ctx); + + bool end_of_context; + size_t output_size; + size_t max_output_size = 2; + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + max_output_size, &end_of_context)); + GPR_ASSERT(output_size = max_output_size); + GPR_ASSERT(end_of_context == false); + grpc_slice slice_recv = grpc_slice_buffer_take_first(&sink); + char *str_recv = (char *)GRPC_SLICE_START_PTR(slice_recv); + GPR_ASSERT(GRPC_SLICE_LENGTH(slice_recv) == max_output_size); + GPR_ASSERT(0 == strncmp(test_str, str_recv, max_output_size)); + grpc_slice_unref(slice_recv); + + size_t remaining_size = sizeof(test_str) - 1 - max_output_size; + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + remaining_size, &end_of_context)); + GPR_ASSERT(output_size = remaining_size); + GPR_ASSERT(end_of_context = true); + + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str + max_output_size)); + + grpc_stream_compression_context_destroy(decompress_ctx); + grpc_slice_buffer_destroy(&source); + grpc_slice_buffer_destroy(&relay); + grpc_slice_buffer_destroy(&sink); +} + +#define LARGE_DATA_SIZE 1024 * 1024 +static void test_stream_compression_simple_compress_decompress_with_large_data() { + char test_str[LARGE_DATA_SIZE]; + generate_random_payload(test_str, LARGE_DATA_SIZE); + grpc_slice_buffer source, relay, sink; + grpc_slice_buffer_init(&source); + grpc_slice_buffer_init(&relay); + grpc_slice_buffer_init(&sink); + grpc_stream_compression_context *compress_ctx = + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_stream_compression_context *decompress_ctx = + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_DECOMPRESS); + grpc_slice slice = grpc_slice_from_static_string(test_str); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + bool end_of_context; + size_t output_size; + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + ~(size_t)0, &end_of_context)); + GPR_ASSERT(output_size = sizeof(test_str) - 1); + grpc_stream_compression_context_destroy(compress_ctx); + grpc_stream_compression_context_destroy(decompress_ctx); + + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str)); + + grpc_slice_buffer_destroy(&source); + grpc_slice_buffer_destroy(&relay); + grpc_slice_buffer_destroy(&sink); +} + +static void test_stream_compression_drop_context() { + const char test_str[] = "aaaaaaabbbbbbbccccccc"; + const char test_str2[] = "dddddddeeeeeeefffffffggggg"; + grpc_slice_buffer source, relay, sink; + grpc_slice_buffer_init(&source); + grpc_slice_buffer_init(&relay); + grpc_slice_buffer_init(&sink); + grpc_stream_compression_context *compress_ctx = + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_slice slice = grpc_slice_from_static_string(test_str); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + grpc_stream_compression_context_destroy(compress_ctx); + + compress_ctx = + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + slice = grpc_slice_from_static_string(test_str2); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + grpc_stream_compression_context_destroy(compress_ctx); + + /* Concatenate the two compressed sliced into one to test decompressing two + * contexts */ + grpc_slice slice1 = grpc_slice_buffer_take_first(&relay); + grpc_slice slice2 = grpc_slice_buffer_take_first(&relay); + grpc_slice slice3 = + grpc_slice_malloc(GRPC_SLICE_LENGTH(slice1) + GRPC_SLICE_LENGTH(slice2)); + memcpy(GRPC_SLICE_START_PTR(slice3), GRPC_SLICE_START_PTR(slice1), + GRPC_SLICE_LENGTH(slice1)); + memcpy(GRPC_SLICE_START_PTR(slice3) + GRPC_SLICE_LENGTH(slice1), + GRPC_SLICE_START_PTR(slice2), GRPC_SLICE_LENGTH(slice2)); + grpc_slice_unref(slice1); + grpc_slice_unref(slice2); + grpc_slice_buffer_add(&relay, slice3); + + grpc_stream_compression_context *decompress_ctx = + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_DECOMPRESS); + bool end_of_context; + size_t output_size; + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + ~(size_t)0, &end_of_context)); + GPR_ASSERT(end_of_context == true); + GPR_ASSERT(output_size == sizeof(test_str) - 1); + + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str)); + grpc_stream_compression_context_destroy(decompress_ctx); + grpc_slice_buffer_destroy(&sink); + + grpc_slice_buffer_init(&sink); + decompress_ctx = grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_DECOMPRESS); + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + ~(size_t)0, &end_of_context)); + GPR_ASSERT(end_of_context == true); + GPR_ASSERT(output_size == sizeof(test_str2) - 1); + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str2)); + grpc_stream_compression_context_destroy(decompress_ctx); + + grpc_slice_buffer_destroy(&source); + grpc_slice_buffer_destroy(&relay); + grpc_slice_buffer_destroy(&sink); +} + +static void test_stream_compression_sync_flush() { + const char test_str[] = "aaaaaaabbbbbbbccccccc"; + const char test_str2[] = "dddddddeeeeeeefffffffggggg"; + grpc_slice_buffer source, relay, sink; + grpc_slice_buffer_init(&source); + grpc_slice_buffer_init(&relay); + grpc_slice_buffer_init(&sink); + grpc_stream_compression_context *compress_ctx = + grpc_stream_compression_context_create(GRPC_STREAM_COMPRESSION_COMPRESS); + grpc_slice slice = grpc_slice_from_static_string(test_str); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_SYNC)); + + grpc_stream_compression_context *decompress_ctx = + grpc_stream_compression_context_create( + GRPC_STREAM_COMPRESSION_DECOMPRESS); + bool end_of_context; + size_t output_size; + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + ~(size_t)0, &end_of_context)); + GPR_ASSERT(end_of_context == false); + GPR_ASSERT(output_size == sizeof(test_str) - 1); + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str)); + grpc_slice_buffer_destroy(&sink); + + grpc_slice_buffer_init(&sink); + slice = grpc_slice_from_static_string(test_str2); + grpc_slice_buffer_add(&source, slice); + GPR_ASSERT(grpc_stream_compress(compress_ctx, &source, &relay, NULL, + ~(size_t)0, + GRPC_STREAM_COMPRESSION_FLUSH_FINISH)); + grpc_stream_compression_context_destroy(compress_ctx); + + GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, + ~(size_t)0, &end_of_context)); + GPR_ASSERT(end_of_context == true); + GPR_ASSERT(output_size == sizeof(test_str2) - 1); + GPR_ASSERT(slice_buffer_equals_string(&sink, test_str2)); + grpc_stream_compression_context_destroy(decompress_ctx); + + grpc_slice_buffer_destroy(&source); + grpc_slice_buffer_destroy(&relay); + grpc_slice_buffer_destroy(&sink); +} + +int main(int argc, char **argv) { + grpc_init(); + test_stream_compression_simple_compress_decompress(); + test_stream_compression_simple_compress_decompress_with_output_size_constraint(); + test_stream_compression_simple_compress_decompress_with_large_data(); + test_stream_compression_sync_flush(); + test_stream_compression_drop_context(); + grpc_shutdown(); + + return 0; +} diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 63067b3081f..afd66ad5bd2 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1041,6 +1041,8 @@ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.h \ src/core/ext/transport/chttp2/transport/internal.h \ src/core/ext/transport/chttp2/transport/parsing.c \ +src/core/ext/transport/chttp2/transport/stream_compression.c \ +src/core/ext/transport/chttp2/transport/stream_compression.h \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/stream_map.h \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 0c4e1fae1d8..c48d8802458 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -2148,6 +2148,23 @@ "third_party": false, "type": "target" }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "is_filegroup": false, + "language": "c", + "name": "stream_compression_test", + "src": [ + "test/core/compression/stream_compression_test.c" + ], + "third_party": false, + "type": "target" + }, { "deps": [ "gpr", @@ -8547,6 +8564,7 @@ "src/core/ext/transport/chttp2/transport/huffsyms.h", "src/core/ext/transport/chttp2/transport/incoming_metadata.h", "src/core/ext/transport/chttp2/transport/internal.h", + "src/core/ext/transport/chttp2/transport/stream_compression.h", "src/core/ext/transport/chttp2/transport/stream_map.h", "src/core/ext/transport/chttp2/transport/varint.h" ], @@ -8588,6 +8606,8 @@ "src/core/ext/transport/chttp2/transport/incoming_metadata.h", "src/core/ext/transport/chttp2/transport/internal.h", "src/core/ext/transport/chttp2/transport/parsing.c", + "src/core/ext/transport/chttp2/transport/stream_compression.c", + "src/core/ext/transport/chttp2/transport/stream_compression.h", "src/core/ext/transport/chttp2/transport/stream_lists.c", "src/core/ext/transport/chttp2/transport/stream_map.c", "src/core/ext/transport/chttp2/transport/stream_map.h", diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 8cca6d3dafb..bc09180f322 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -2249,6 +2249,28 @@ "windows" ] }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "exclude_iomgrs": [], + "flaky": false, + "gtest": false, + "language": "c", + "name": "stream_compression_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, { "args": [], "ci_platforms": [ diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 3c6e8d8f34f..5f145ddcc59 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -1552,6 +1552,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "status_conversion_test", "v {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "stream_compression_test", "vcxproj\test\stream_compression_test\stream_compression_test.vcxproj", "{A5EE72A2-656C-0896-12F3-A92583CF7C61}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "stream_owned_slice_test", "vcxproj\test\stream_owned_slice_test\stream_owned_slice_test.vcxproj", "{D5A20C05-D9B2-970B-8429-94BC3F58D1C4}" ProjectSection(myProperties) = preProject lib = "False" @@ -4040,6 +4051,22 @@ Global {21E2A241-9D48-02CD-92E4-4EEC98424CF5}.Release-DLL|Win32.Build.0 = Release|Win32 {21E2A241-9D48-02CD-92E4-4EEC98424CF5}.Release-DLL|x64.ActiveCfg = Release|x64 {21E2A241-9D48-02CD-92E4-4EEC98424CF5}.Release-DLL|x64.Build.0 = Release|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug|Win32.ActiveCfg = Debug|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug|x64.ActiveCfg = Debug|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release|Win32.ActiveCfg = Release|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release|x64.ActiveCfg = Release|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug|Win32.Build.0 = Debug|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug|x64.Build.0 = Debug|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release|Win32.Build.0 = Release|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release|x64.Build.0 = Release|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Debug-DLL|x64.Build.0 = Debug|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release-DLL|Win32.Build.0 = Release|Win32 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release-DLL|x64.ActiveCfg = Release|x64 + {A5EE72A2-656C-0896-12F3-A92583CF7C61}.Release-DLL|x64.Build.0 = Release|x64 {D5A20C05-D9B2-970B-8429-94BC3F58D1C4}.Debug|Win32.ActiveCfg = Debug|Win32 {D5A20C05-D9B2-970B-8429-94BC3F58D1C4}.Debug|x64.ActiveCfg = Debug|x64 {D5A20C05-D9B2-970B-8429-94BC3F58D1C4}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index f9badbbd353..f6f2a1ea363 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -428,6 +428,7 @@ + @@ -807,6 +808,8 @@ + + diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index f3e997e4e0e..3054c9f7452 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -433,6 +433,9 @@ src\core\ext\transport\chttp2\transport + + src\core\ext\transport\chttp2\transport + src\core\ext\transport\chttp2\transport @@ -1229,6 +1232,9 @@ src\core\ext\transport\chttp2\transport + + src\core\ext\transport\chttp2\transport + src\core\ext\transport\chttp2\transport diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 3a865f3e812..9fb0b9983c6 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -418,6 +418,7 @@ + @@ -776,6 +777,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 0310ebc012c..e9ca820fecf 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -439,6 +439,9 @@ src\core\ext\transport\chttp2\transport + + src\core\ext\transport\chttp2\transport + src\core\ext\transport\chttp2\transport @@ -1139,6 +1142,9 @@ src\core\ext\transport\chttp2\transport + + src\core\ext\transport\chttp2\transport + src\core\ext\transport\chttp2\transport diff --git a/vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj b/vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj new file mode 100644 index 00000000000..7faf665b20d --- /dev/null +++ b/vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj @@ -0,0 +1,199 @@ + + + + + + Debug + Win32 + + + Debug + x64 + + + Release + Win32 + + + Release + x64 + + + + {A5EE72A2-656C-0896-12F3-A92583CF7C61} + true + $(SolutionDir)IntDir\$(MSBuildProjectName)\ + + + + v100 + + + v110 + + + v120 + + + v140 + + + Application + true + Unicode + + + Application + false + true + Unicode + + + + + + + + + + + + + + stream_compression_test + static + Debug + static + Debug + + + stream_compression_test + static + Release + static + Release + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + Disabled + WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions) + true + MultiThreadedDebug + true + None + false + + + Console + true + false + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + NotUsing + Level3 + MaxSpeed + WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions) + true + true + true + MultiThreaded + true + None + false + + + Console + true + false + true + true + + + + + + + + + + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + + + {29D16885-7228-4C31-81ED-5F9187C7F2A9} + + + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + + + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + + + diff --git a/vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj.filters b/vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj.filters new file mode 100644 index 00000000000..a63628279da --- /dev/null +++ b/vsprojects/vcxproj/test/stream_compression_test/stream_compression_test.vcxproj.filters @@ -0,0 +1,21 @@ + + + + + test\core\compression + + + + + + {22ec1dc6-29e1-32ac-1494-43bb7f211422} + + + {62cac7ff-76a5-35ff-1e73-a8508e826ba3} + + + {02c8e4fc-eeda-2f58-227f-ebef22a39562} + + + + From e87a7e1a4736a4e802e5eaa0f14b5c8c956f5f76 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 29 Jun 2017 16:53:24 -0700 Subject: [PATCH 02/10] Move stream_compression to src/core/lib/compression --- CMakeLists.txt | 9 +++++---- Makefile | 9 +++++---- binding.gyp | 2 +- build.yaml | 4 ++-- config.m4 | 2 +- config.w32 | 2 +- gRPC-Core.podspec | 6 +++--- grpc.gemspec | 4 ++-- package.xml | 4 ++-- .../transport/chttp2/transport/chttp2_transport.c | 1 + src/core/ext/transport/chttp2/transport/internal.h | 1 + .../compression}/stream_compression.c | 2 +- .../compression}/stream_compression.h | 0 src/python/grpcio/grpc_core_dependencies.py | 2 +- test/core/compression/stream_compression_test.c | 2 +- tools/doxygen/Doxyfile.core.internal | 4 ++-- tools/run_tests/generated/sources_and_headers.json | 6 +++--- vsprojects/vcxproj/grpc/grpc.vcxproj | 6 +++--- vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 12 ++++++------ .../vcxproj/grpc_test_util/grpc_test_util.vcxproj | 3 +++ .../grpc_test_util/grpc_test_util.vcxproj.filters | 6 ++++++ .../vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 6 +++--- .../grpc_unsecure/grpc_unsecure.vcxproj.filters | 12 ++++++------ 23 files changed, 59 insertions(+), 46 deletions(-) rename src/core/{ext/transport/chttp2/transport => lib/compression}/stream_compression.c (99%) rename src/core/{ext/transport/chttp2/transport => lib/compression}/stream_compression.h (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9aae4f06bd..c602813611f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -934,6 +934,7 @@ add_library(grpc src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c + src/core/lib/compression/stream_compression.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1068,7 +1069,6 @@ add_library(grpc src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c - src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -1274,6 +1274,7 @@ add_library(grpc_cronet src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c + src/core/lib/compression/stream_compression.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1411,7 +1412,6 @@ add_library(grpc_cronet src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c - src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -1594,6 +1594,7 @@ add_library(grpc_test_util src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c + src/core/lib/compression/stream_compression.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1856,6 +1857,7 @@ add_library(grpc_unsecure src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c + src/core/lib/compression/stream_compression.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c @@ -1991,7 +1993,6 @@ add_library(grpc_unsecure src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c - src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -2500,7 +2501,6 @@ add_library(grpc++_cronet src/core/ext/transport/chttp2/transport/huffsyms.c src/core/ext/transport/chttp2/transport/incoming_metadata.c src/core/ext/transport/chttp2/transport/parsing.c - src/core/ext/transport/chttp2/transport/stream_compression.c src/core/ext/transport/chttp2/transport/stream_lists.c src/core/ext/transport/chttp2/transport/stream_map.c src/core/ext/transport/chttp2/transport/varint.c @@ -2514,6 +2514,7 @@ add_library(grpc++_cronet src/core/lib/channel/handshaker_registry.c src/core/lib/compression/compression.c src/core/lib/compression/message_compress.c + src/core/lib/compression/stream_compression.c src/core/lib/http/format_request.c src/core/lib/http/httpcli.c src/core/lib/http/parser.c diff --git a/Makefile b/Makefile index db9e050f87b..1e9ee07e6b8 100644 --- a/Makefile +++ b/Makefile @@ -2886,6 +2886,7 @@ LIBGRPC_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ + src/core/lib/compression/stream_compression.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3020,7 +3021,6 @@ LIBGRPC_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ - src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -3224,6 +3224,7 @@ LIBGRPC_CRONET_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ + src/core/lib/compression/stream_compression.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3361,7 +3362,6 @@ LIBGRPC_CRONET_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ - src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -3541,6 +3541,7 @@ LIBGRPC_TEST_UTIL_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ + src/core/lib/compression/stream_compression.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3775,6 +3776,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ + src/core/lib/compression/stream_compression.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -3910,7 +3912,6 @@ LIBGRPC_UNSECURE_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ - src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -4403,7 +4404,6 @@ LIBGRPC++_CRONET_SRC = \ src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ - src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ @@ -4417,6 +4417,7 @@ LIBGRPC++_CRONET_SRC = \ src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ + src/core/lib/compression/stream_compression.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ diff --git a/binding.gyp b/binding.gyp index 11c06941f70..92581a8b999 100644 --- a/binding.gyp +++ b/binding.gyp @@ -657,6 +657,7 @@ 'src/core/lib/channel/handshaker_registry.c', 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', + 'src/core/lib/compression/stream_compression.c', 'src/core/lib/http/format_request.c', 'src/core/lib/http/httpcli.c', 'src/core/lib/http/parser.c', @@ -791,7 +792,6 @@ 'src/core/ext/transport/chttp2/transport/huffsyms.c', 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', 'src/core/ext/transport/chttp2/transport/parsing.c', - 'src/core/ext/transport/chttp2/transport/stream_compression.c', 'src/core/ext/transport/chttp2/transport/stream_lists.c', 'src/core/ext/transport/chttp2/transport/stream_map.c', 'src/core/ext/transport/chttp2/transport/varint.c', diff --git a/build.yaml b/build.yaml index ee71e3fa8ae..6de33cf1746 100644 --- a/build.yaml +++ b/build.yaml @@ -189,6 +189,7 @@ filegroups: - src/core/lib/channel/handshaker_registry.h - src/core/lib/compression/algorithm_metadata.h - src/core/lib/compression/message_compress.h + - src/core/lib/compression/stream_compression.h - src/core/lib/http/format_request.h - src/core/lib/http/httpcli.h - src/core/lib/http/parser.h @@ -298,6 +299,7 @@ filegroups: - src/core/lib/channel/handshaker_registry.c - src/core/lib/compression/compression.c - src/core/lib/compression/message_compress.c + - src/core/lib/compression/stream_compression.c - src/core/lib/http/format_request.c - src/core/lib/http/httpcli.c - src/core/lib/http/parser.c @@ -738,7 +740,6 @@ filegroups: - src/core/ext/transport/chttp2/transport/huffsyms.h - src/core/ext/transport/chttp2/transport/incoming_metadata.h - src/core/ext/transport/chttp2/transport/internal.h - - src/core/ext/transport/chttp2/transport/stream_compression.h - src/core/ext/transport/chttp2/transport/stream_map.h - src/core/ext/transport/chttp2/transport/varint.h src: @@ -759,7 +760,6 @@ filegroups: - src/core/ext/transport/chttp2/transport/huffsyms.c - src/core/ext/transport/chttp2/transport/incoming_metadata.c - src/core/ext/transport/chttp2/transport/parsing.c - - src/core/ext/transport/chttp2/transport/stream_compression.c - src/core/ext/transport/chttp2/transport/stream_lists.c - src/core/ext/transport/chttp2/transport/stream_map.c - src/core/ext/transport/chttp2/transport/varint.c diff --git a/config.m4 b/config.m4 index 6bf72eea8ad..18563094561 100644 --- a/config.m4 +++ b/config.m4 @@ -94,6 +94,7 @@ if test "$PHP_GRPC" != "no"; then src/core/lib/channel/handshaker_registry.c \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ + src/core/lib/compression/stream_compression.c \ src/core/lib/http/format_request.c \ src/core/lib/http/httpcli.c \ src/core/lib/http/parser.c \ @@ -228,7 +229,6 @@ if test "$PHP_GRPC" != "no"; then src/core/ext/transport/chttp2/transport/huffsyms.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/parsing.c \ - src/core/ext/transport/chttp2/transport/stream_compression.c \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/varint.c \ diff --git a/config.w32 b/config.w32 index 46779e0723d..5a8381d81a1 100644 --- a/config.w32 +++ b/config.w32 @@ -71,6 +71,7 @@ if (PHP_GRPC != "no") { "src\\core\\lib\\channel\\handshaker_registry.c " + "src\\core\\lib\\compression\\compression.c " + "src\\core\\lib\\compression\\message_compress.c " + + "src\\core\\lib\\compression\\stream_compression.c " + "src\\core\\lib\\http\\format_request.c " + "src\\core\\lib\\http\\httpcli.c " + "src\\core\\lib\\http\\parser.c " + @@ -205,7 +206,6 @@ if (PHP_GRPC != "no") { "src\\core\\ext\\transport\\chttp2\\transport\\huffsyms.c " + "src\\core\\ext\\transport\\chttp2\\transport\\incoming_metadata.c " + "src\\core\\ext\\transport\\chttp2\\transport\\parsing.c " + - "src\\core\\ext\\transport\\chttp2\\transport\\stream_compression.c " + "src\\core\\ext\\transport\\chttp2\\transport\\stream_lists.c " + "src\\core\\ext\\transport\\chttp2\\transport\\stream_map.c " + "src\\core\\ext\\transport\\chttp2\\transport\\varint.c " + diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index 0062bf9f145..125b7e04d2f 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -253,6 +253,7 @@ Pod::Spec.new do |s| 'src/core/lib/channel/handshaker_registry.h', 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', + 'src/core/lib/compression/stream_compression.h', 'src/core/lib/http/format_request.h', 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', @@ -370,7 +371,6 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/huffsyms.h', 'src/core/ext/transport/chttp2/transport/incoming_metadata.h', 'src/core/ext/transport/chttp2/transport/internal.h', - 'src/core/ext/transport/chttp2/transport/stream_compression.h', 'src/core/ext/transport/chttp2/transport/stream_map.h', 'src/core/ext/transport/chttp2/transport/varint.h', 'src/core/ext/transport/chttp2/alpn/alpn.h', @@ -470,6 +470,7 @@ Pod::Spec.new do |s| 'src/core/lib/channel/handshaker_registry.c', 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', + 'src/core/lib/compression/stream_compression.c', 'src/core/lib/http/format_request.c', 'src/core/lib/http/httpcli.c', 'src/core/lib/http/parser.c', @@ -604,7 +605,6 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/huffsyms.c', 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', 'src/core/ext/transport/chttp2/transport/parsing.c', - 'src/core/ext/transport/chttp2/transport/stream_compression.c', 'src/core/ext/transport/chttp2/transport/stream_lists.c', 'src/core/ext/transport/chttp2/transport/stream_map.c', 'src/core/ext/transport/chttp2/transport/varint.c', @@ -738,6 +738,7 @@ Pod::Spec.new do |s| 'src/core/lib/channel/handshaker_registry.h', 'src/core/lib/compression/algorithm_metadata.h', 'src/core/lib/compression/message_compress.h', + 'src/core/lib/compression/stream_compression.h', 'src/core/lib/http/format_request.h', 'src/core/lib/http/httpcli.h', 'src/core/lib/http/parser.h', @@ -855,7 +856,6 @@ Pod::Spec.new do |s| 'src/core/ext/transport/chttp2/transport/huffsyms.h', 'src/core/ext/transport/chttp2/transport/incoming_metadata.h', 'src/core/ext/transport/chttp2/transport/internal.h', - 'src/core/ext/transport/chttp2/transport/stream_compression.h', 'src/core/ext/transport/chttp2/transport/stream_map.h', 'src/core/ext/transport/chttp2/transport/varint.h', 'src/core/ext/transport/chttp2/alpn/alpn.h', diff --git a/grpc.gemspec b/grpc.gemspec index daf3c99702c..82c0a2417af 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -184,6 +184,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/channel/handshaker_registry.h ) s.files += %w( src/core/lib/compression/algorithm_metadata.h ) s.files += %w( src/core/lib/compression/message_compress.h ) + s.files += %w( src/core/lib/compression/stream_compression.h ) s.files += %w( src/core/lib/http/format_request.h ) s.files += %w( src/core/lib/http/httpcli.h ) s.files += %w( src/core/lib/http/parser.h ) @@ -301,7 +302,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/transport/huffsyms.h ) s.files += %w( src/core/ext/transport/chttp2/transport/incoming_metadata.h ) s.files += %w( src/core/ext/transport/chttp2/transport/internal.h ) - s.files += %w( src/core/ext/transport/chttp2/transport/stream_compression.h ) s.files += %w( src/core/ext/transport/chttp2/transport/stream_map.h ) s.files += %w( src/core/ext/transport/chttp2/transport/varint.h ) s.files += %w( src/core/ext/transport/chttp2/alpn/alpn.h ) @@ -401,6 +401,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/channel/handshaker_registry.c ) s.files += %w( src/core/lib/compression/compression.c ) s.files += %w( src/core/lib/compression/message_compress.c ) + s.files += %w( src/core/lib/compression/stream_compression.c ) s.files += %w( src/core/lib/http/format_request.c ) s.files += %w( src/core/lib/http/httpcli.c ) s.files += %w( src/core/lib/http/parser.c ) @@ -535,7 +536,6 @@ Gem::Specification.new do |s| s.files += %w( src/core/ext/transport/chttp2/transport/huffsyms.c ) s.files += %w( src/core/ext/transport/chttp2/transport/incoming_metadata.c ) s.files += %w( src/core/ext/transport/chttp2/transport/parsing.c ) - s.files += %w( src/core/ext/transport/chttp2/transport/stream_compression.c ) s.files += %w( src/core/ext/transport/chttp2/transport/stream_lists.c ) s.files += %w( src/core/ext/transport/chttp2/transport/stream_map.c ) s.files += %w( src/core/ext/transport/chttp2/transport/varint.c ) diff --git a/package.xml b/package.xml index 1e8f408c218..c57bab941ed 100644 --- a/package.xml +++ b/package.xml @@ -198,6 +198,7 @@ + @@ -315,7 +316,6 @@ - @@ -415,6 +415,7 @@ + @@ -549,7 +550,6 @@ - diff --git a/src/core/ext/transport/chttp2/transport/chttp2_transport.c b/src/core/ext/transport/chttp2/transport/chttp2_transport.c index cd788e9ce34..ab7dc3ba9ff 100644 --- a/src/core/ext/transport/chttp2/transport/chttp2_transport.c +++ b/src/core/ext/transport/chttp2/transport/chttp2_transport.c @@ -33,6 +33,7 @@ #include "src/core/ext/transport/chttp2/transport/internal.h" #include "src/core/ext/transport/chttp2/transport/varint.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/compression/stream_compression.h" #include "src/core/lib/http/parser.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/profiling/timers.h" diff --git a/src/core/ext/transport/chttp2/transport/internal.h b/src/core/ext/transport/chttp2/transport/internal.h index 9fa72ddbdf8..b0726fdb5fd 100644 --- a/src/core/ext/transport/chttp2/transport/internal.h +++ b/src/core/ext/transport/chttp2/transport/internal.h @@ -33,6 +33,7 @@ #include "src/core/ext/transport/chttp2/transport/hpack_parser.h" #include "src/core/ext/transport/chttp2/transport/incoming_metadata.h" #include "src/core/ext/transport/chttp2/transport/stream_map.h" +#include "src/core/lib/compression/stream_compression.h" #include "src/core/lib/iomgr/combiner.h" #include "src/core/lib/iomgr/endpoint.h" #include "src/core/lib/iomgr/timer.h" diff --git a/src/core/ext/transport/chttp2/transport/stream_compression.c b/src/core/lib/compression/stream_compression.c similarity index 99% rename from src/core/ext/transport/chttp2/transport/stream_compression.c rename to src/core/lib/compression/stream_compression.c index b2869102372..e09e838e8e4 100644 --- a/src/core/ext/transport/chttp2/transport/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -34,7 +34,7 @@ #include #include -#include "src/core/ext/transport/chttp2/transport/stream_compression.h" +#include "src/core/lib/compression/stream_compression.h" #include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/slice/slice_internal.h" diff --git a/src/core/ext/transport/chttp2/transport/stream_compression.h b/src/core/lib/compression/stream_compression.h similarity index 100% rename from src/core/ext/transport/chttp2/transport/stream_compression.h rename to src/core/lib/compression/stream_compression.h diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index aa7e1b53178..cf3d5256e61 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -70,6 +70,7 @@ CORE_SOURCE_FILES = [ 'src/core/lib/channel/handshaker_registry.c', 'src/core/lib/compression/compression.c', 'src/core/lib/compression/message_compress.c', + 'src/core/lib/compression/stream_compression.c', 'src/core/lib/http/format_request.c', 'src/core/lib/http/httpcli.c', 'src/core/lib/http/parser.c', @@ -204,7 +205,6 @@ CORE_SOURCE_FILES = [ 'src/core/ext/transport/chttp2/transport/huffsyms.c', 'src/core/ext/transport/chttp2/transport/incoming_metadata.c', 'src/core/ext/transport/chttp2/transport/parsing.c', - 'src/core/ext/transport/chttp2/transport/stream_compression.c', 'src/core/ext/transport/chttp2/transport/stream_lists.c', 'src/core/ext/transport/chttp2/transport/stream_map.c', 'src/core/ext/transport/chttp2/transport/varint.c', diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index 28700c8f703..da931ef8e26 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -38,7 +38,7 @@ #include #include -#include "src/core/ext/transport/chttp2/transport/stream_compression.h" +#include "src/core/lib/compression/stream_compression.h" static void generate_random_payload(char *payload, size_t size) { size_t i; diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index afd66ad5bd2..e951862bacf 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1041,8 +1041,6 @@ src/core/ext/transport/chttp2/transport/incoming_metadata.c \ src/core/ext/transport/chttp2/transport/incoming_metadata.h \ src/core/ext/transport/chttp2/transport/internal.h \ src/core/ext/transport/chttp2/transport/parsing.c \ -src/core/ext/transport/chttp2/transport/stream_compression.c \ -src/core/ext/transport/chttp2/transport/stream_compression.h \ src/core/ext/transport/chttp2/transport/stream_lists.c \ src/core/ext/transport/chttp2/transport/stream_map.c \ src/core/ext/transport/chttp2/transport/stream_map.h \ @@ -1070,6 +1068,8 @@ src/core/lib/compression/algorithm_metadata.h \ src/core/lib/compression/compression.c \ src/core/lib/compression/message_compress.c \ src/core/lib/compression/message_compress.h \ +src/core/lib/compression/stream_compression.c \ +src/core/lib/compression/stream_compression.h \ src/core/lib/debug/trace.c \ src/core/lib/debug/trace.h \ src/core/lib/http/format_request.c \ diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index c48d8802458..67f05739086 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -7629,6 +7629,7 @@ "src/core/lib/channel/handshaker_registry.h", "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", + "src/core/lib/compression/stream_compression.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h", @@ -7763,6 +7764,8 @@ "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", "src/core/lib/compression/message_compress.h", + "src/core/lib/compression/stream_compression.c", + "src/core/lib/compression/stream_compression.h", "src/core/lib/http/format_request.c", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.c", @@ -8564,7 +8567,6 @@ "src/core/ext/transport/chttp2/transport/huffsyms.h", "src/core/ext/transport/chttp2/transport/incoming_metadata.h", "src/core/ext/transport/chttp2/transport/internal.h", - "src/core/ext/transport/chttp2/transport/stream_compression.h", "src/core/ext/transport/chttp2/transport/stream_map.h", "src/core/ext/transport/chttp2/transport/varint.h" ], @@ -8606,8 +8608,6 @@ "src/core/ext/transport/chttp2/transport/incoming_metadata.h", "src/core/ext/transport/chttp2/transport/internal.h", "src/core/ext/transport/chttp2/transport/parsing.c", - "src/core/ext/transport/chttp2/transport/stream_compression.c", - "src/core/ext/transport/chttp2/transport/stream_compression.h", "src/core/ext/transport/chttp2/transport/stream_lists.c", "src/core/ext/transport/chttp2/transport/stream_map.c", "src/core/ext/transport/chttp2/transport/stream_map.h", diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index f6f2a1ea363..5c6d48f9a9d 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -311,6 +311,7 @@ + @@ -428,7 +429,6 @@ - @@ -540,6 +540,8 @@ + + @@ -808,8 +810,6 @@ - - diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index 3054c9f7452..2057e2f9bfa 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -31,6 +31,9 @@ src\core\lib\compression + + src\core\lib\compression + src\core\lib\http @@ -433,9 +436,6 @@ src\core\ext\transport\chttp2\transport - - src\core\ext\transport\chttp2\transport - src\core\ext\transport\chttp2\transport @@ -881,6 +881,9 @@ src\core\lib\compression + + src\core\lib\compression + src\core\lib\http @@ -1232,9 +1235,6 @@ src\core\ext\transport\chttp2\transport - - src\core\ext\transport\chttp2\transport - src\core\ext\transport\chttp2\transport diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj index 1899be91db3..32809ca5899 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj @@ -206,6 +206,7 @@ + @@ -366,6 +367,8 @@ + + diff --git a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters index 9c81fc5bf51..def8de42028 100644 --- a/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_test_util/grpc_test_util.vcxproj.filters @@ -88,6 +88,9 @@ src\core\lib\compression + + src\core\lib\compression + src\core\lib\http @@ -611,6 +614,9 @@ src\core\lib\compression + + src\core\lib\compression + src\core\lib\http diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 9fb0b9983c6..1c7cb7ae62e 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -301,6 +301,7 @@ + @@ -418,7 +419,6 @@ - @@ -507,6 +507,8 @@ + + @@ -777,8 +779,6 @@ - - diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index e9ca820fecf..641bdec0dba 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -34,6 +34,9 @@ src\core\lib\compression + + src\core\lib\compression + src\core\lib\http @@ -439,9 +442,6 @@ src\core\ext\transport\chttp2\transport - - src\core\ext\transport\chttp2\transport - src\core\ext\transport\chttp2\transport @@ -791,6 +791,9 @@ src\core\lib\compression + + src\core\lib\compression + src\core\lib\http @@ -1142,9 +1145,6 @@ src\core\ext\transport\chttp2\transport - - src\core\ext\transport\chttp2\transport - src\core\ext\transport\chttp2\transport From 163d8d65a68925944e5e16bb91d674a5303b5cc4 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 30 Jun 2017 09:53:37 -0700 Subject: [PATCH 03/10] Sanity fix --- src/core/lib/compression/stream_compression.c | 35 +++++------------ src/core/lib/compression/stream_compression.h | 39 ++++++------------- .../compression/stream_compression_test.c | 35 +++++------------ 3 files changed, 32 insertions(+), 77 deletions(-) diff --git a/src/core/lib/compression/stream_compression.c b/src/core/lib/compression/stream_compression.c index e09e838e8e4..df13d53e06f 100644 --- a/src/core/lib/compression/stream_compression.c +++ b/src/core/lib/compression/stream_compression.c @@ -1,33 +1,18 @@ /* * - * Copyright 2017, Google Inc. - * All rights reserved. + * Copyright 2017 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index 21ae2fd32d1..41a7b669f72 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -1,38 +1,23 @@ /* * - * Copyright 2017, Google Inc. - * All rights reserved. + * Copyright 2017 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ -#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRONSPORT_STREAM_COMPRESSION_H -#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRONSPORT_STREAM_COMPRESSION_H +#ifndef GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_H +#define GRPC_CORE_LIB_COMPRESSION_STREAM_COMPRESSION_H #include diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index da931ef8e26..b2563ca2942 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -1,33 +1,18 @@ /* * - * Copyright 2017, Google Inc. - * All rights reserved. + * Copyright 2017 gRPC authors. * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google Inc. nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. + * http://www.apache.org/licenses/LICENSE-2.0 * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. * */ From 4f85558ba829e5ad072f54d58b49de7710ba585c Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 5 Jul 2017 10:37:54 -0700 Subject: [PATCH 04/10] clang-format --- test/core/compression/stream_compression_test.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index b2563ca2942..e318020245b 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -20,8 +20,8 @@ #include #include -#include #include +#include #include "src/core/lib/compression/stream_compression.h" @@ -34,7 +34,8 @@ static void generate_random_payload(char *payload, size_t size) { payload[size - 1] = '\0'; } -static bool slice_buffer_equals_string(grpc_slice_buffer *buf, const char *str) { +static bool slice_buffer_equals_string(grpc_slice_buffer *buf, + const char *str) { size_t i; if (buf->length != strlen(str)) { return false; @@ -42,7 +43,8 @@ static bool slice_buffer_equals_string(grpc_slice_buffer *buf, const char *str) size_t pointer = 0; for (i = 0; i < buf->count; i++) { size_t slice_len = GRPC_SLICE_LENGTH(buf->slices[i]); - if (0 != strncmp(str + pointer, (char *)GRPC_SLICE_START_PTR(buf->slices[i]), slice_len)) { + if (0 != strncmp(str + pointer, + (char *)GRPC_SLICE_START_PTR(buf->slices[i]), slice_len)) { return false; } pointer += slice_len; @@ -128,7 +130,8 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() } #define LARGE_DATA_SIZE 1024 * 1024 -static void test_stream_compression_simple_compress_decompress_with_large_data() { +static void +test_stream_compression_simple_compress_decompress_with_large_data() { char test_str[LARGE_DATA_SIZE]; generate_random_payload(test_str, LARGE_DATA_SIZE); grpc_slice_buffer source, relay, sink; From fbd06f720297b4e622eabf4d1d8f7cdcbc3fde5a Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sat, 8 Jul 2017 10:54:46 -0700 Subject: [PATCH 05/10] Comments and fixes --- src/core/lib/compression/stream_compression.h | 13 ++++++++++--- test/core/compression/stream_compression_test.c | 10 +++++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index 41a7b669f72..cf009fdadd4 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -31,14 +31,15 @@ typedef struct grpc_stream_compression_context { } grpc_stream_compression_context; typedef enum grpc_stream_compression_method { - GRPC_STREAM_COMPRESSION_COMPRESS, + GRPC_STREAM_COMPRESSION_COMPRESS = 0, GRPC_STREAM_COMPRESSION_DECOMPRESS } grpc_stream_compression_method; typedef enum grpc_stream_compression_flush { - GRPC_STREAM_COMPRESSION_FLUSH_NONE, + GRPC_STREAM_COMPRESSION_FLUSH_NONE = 0, GRPC_STREAM_COMPRESSION_FLUSH_SYNC, - GRPC_STREAM_COMPRESSION_FLUSH_FINISH + GRPC_STREAM_COMPRESSION_FLUSH_FINISH, + GRPC_STREAM_COMPRESSION_FLUSH_COUNT } grpc_stream_compression_flush; /** @@ -47,6 +48,12 @@ typedef enum grpc_stream_compression_flush { * into \a out. If all the bytes in input buffer \a in are depleted and \a flush * is not GRPC_STREAM_COMPRESSION_FLUSH_NONE, the corresponding flush method is * executed. The total number of bytes emitted is outputed in \a output_size. + * + * A SYNC flush indicates that the entire messages in \a in can be decompressed + * from \a out. A FINISH flush implies a SYNC flush, and that any further + * compression will not be dependent on the state of the current context and any + * previous compressed bytes. It allows corresponding decompression context to + * be dropped when reaching this boundary. */ bool grpc_stream_compress(grpc_stream_compression_context *ctx, grpc_slice_buffer *in, grpc_slice_buffer *out, diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index e318020245b..5c025e96abe 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -72,7 +72,7 @@ static void test_stream_compression_simple_compress_decompress() { size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, ~(size_t)0, &end_of_context)); - GPR_ASSERT(output_size = sizeof(test_str) - 1); + GPR_ASSERT(output_size == sizeof(test_str) - 1); grpc_stream_compression_context_destroy(compress_ctx); grpc_stream_compression_context_destroy(decompress_ctx); @@ -107,7 +107,7 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() size_t max_output_size = 2; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, max_output_size, &end_of_context)); - GPR_ASSERT(output_size = max_output_size); + GPR_ASSERT(output_size == max_output_size); GPR_ASSERT(end_of_context == false); grpc_slice slice_recv = grpc_slice_buffer_take_first(&sink); char *str_recv = (char *)GRPC_SLICE_START_PTR(slice_recv); @@ -118,8 +118,8 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() size_t remaining_size = sizeof(test_str) - 1 - max_output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, remaining_size, &end_of_context)); - GPR_ASSERT(output_size = remaining_size); - GPR_ASSERT(end_of_context = true); + GPR_ASSERT(output_size == remaining_size); + GPR_ASSERT(end_of_context == true); GPR_ASSERT(slice_buffer_equals_string(&sink, test_str + max_output_size)); @@ -152,7 +152,7 @@ test_stream_compression_simple_compress_decompress_with_large_data() { size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, ~(size_t)0, &end_of_context)); - GPR_ASSERT(output_size = sizeof(test_str) - 1); + GPR_ASSERT(output_size == sizeof(test_str) - 1); grpc_stream_compression_context_destroy(compress_ctx); grpc_stream_compression_context_destroy(decompress_ctx); From dfe402781ec62c7fd15f63423fc89e307839af5c Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Sun, 9 Jul 2017 17:58:35 +0000 Subject: [PATCH 06/10] Fix windows test failure --- test/core/compression/stream_compression_test.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/core/compression/stream_compression_test.c b/test/core/compression/stream_compression_test.c index 5c025e96abe..e576507aaf6 100644 --- a/test/core/compression/stream_compression_test.c +++ b/test/core/compression/stream_compression_test.c @@ -129,10 +129,10 @@ test_stream_compression_simple_compress_decompress_with_output_size_constraint() grpc_slice_buffer_destroy(&sink); } -#define LARGE_DATA_SIZE 1024 * 1024 +#define LARGE_DATA_SIZE (1024 * 1024) static void test_stream_compression_simple_compress_decompress_with_large_data() { - char test_str[LARGE_DATA_SIZE]; + char *test_str = gpr_malloc(LARGE_DATA_SIZE * sizeof(char)); generate_random_payload(test_str, LARGE_DATA_SIZE); grpc_slice_buffer source, relay, sink; grpc_slice_buffer_init(&source); @@ -152,7 +152,7 @@ test_stream_compression_simple_compress_decompress_with_large_data() { size_t output_size; GPR_ASSERT(grpc_stream_decompress(decompress_ctx, &relay, &sink, &output_size, ~(size_t)0, &end_of_context)); - GPR_ASSERT(output_size == sizeof(test_str) - 1); + GPR_ASSERT(output_size == LARGE_DATA_SIZE - 1); grpc_stream_compression_context_destroy(compress_ctx); grpc_stream_compression_context_destroy(decompress_ctx); @@ -161,6 +161,7 @@ test_stream_compression_simple_compress_decompress_with_large_data() { grpc_slice_buffer_destroy(&source); grpc_slice_buffer_destroy(&relay); grpc_slice_buffer_destroy(&sink); + gpr_free(test_str); } static void test_stream_compression_drop_context() { From f9ff554fd28065c4de6716620b6cf0a6a57e6113 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Wed, 12 Jul 2017 14:02:27 -0700 Subject: [PATCH 07/10] Add COUNT for stream compression method --- src/core/lib/compression/stream_compression.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/lib/compression/stream_compression.h b/src/core/lib/compression/stream_compression.h index cf009fdadd4..844dff81a3b 100644 --- a/src/core/lib/compression/stream_compression.h +++ b/src/core/lib/compression/stream_compression.h @@ -32,7 +32,8 @@ typedef struct grpc_stream_compression_context { typedef enum grpc_stream_compression_method { GRPC_STREAM_COMPRESSION_COMPRESS = 0, - GRPC_STREAM_COMPRESSION_DECOMPRESS + GRPC_STREAM_COMPRESSION_DECOMPRESS, + GRPC_STREAM_COMPRESSION_METHOD_COUNT } grpc_stream_compression_method; typedef enum grpc_stream_compression_flush { From e8714b93b9c22f1d43a4b71b7e673c812a4fd1a8 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 13 Jul 2017 10:10:30 -0700 Subject: [PATCH 08/10] lib - Fix Bazel build --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index fa423267879..e9b77868ae3 100644 --- a/BUILD +++ b/BUILD @@ -447,6 +447,7 @@ grpc_cc_library( "src/core/lib/channel/handshaker_registry.c", "src/core/lib/compression/compression.c", "src/core/lib/compression/message_compress.c", + "src/core/lib/compression/stream_compression.c", "src/core/lib/http/format_request.c", "src/core/lib/http/httpcli.c", "src/core/lib/http/parser.c", From b750d2b5aa4bc907d32b7691dd9aa00e35ea9a86 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Thu, 13 Jul 2017 11:55:09 -0700 Subject: [PATCH 09/10] lib - Fix ubsan by add exception to zlib function --- tools/ubsan_suppressions.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/ubsan_suppressions.txt b/tools/ubsan_suppressions.txt index 83dcfc3d058..2dcfeea9afa 100644 --- a/tools/ubsan_suppressions.txt +++ b/tools/ubsan_suppressions.txt @@ -7,3 +7,4 @@ alignment:CRYPTO_cbc128_encrypt alignment:CRYPTO_gcm128_encrypt nonnull-attribute:google::protobuf::* alignment:google::protobuf::* +nonnull-attribute:_tr_stored_block From 1e9bdb5d1a9f9293052d8eb53500cc3b3900f2f3 Mon Sep 17 00:00:00 2001 From: Muxi Yan Date: Fri, 14 Jul 2017 10:14:02 -0700 Subject: [PATCH 10/10] lib - fix another Bazel build problem --- BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/BUILD b/BUILD index e9b77868ae3..c9cc6dbd3c0 100644 --- a/BUILD +++ b/BUILD @@ -574,6 +574,7 @@ grpc_cc_library( "src/core/lib/channel/handshaker_registry.h", "src/core/lib/compression/algorithm_metadata.h", "src/core/lib/compression/message_compress.h", + "src/core/lib/compression/stream_compression.h", "src/core/lib/http/format_request.h", "src/core/lib/http/httpcli.h", "src/core/lib/http/parser.h",