From 83bcb0cf2e0110a98a8286dc6cd040ca8ba0c6b0 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Mon, 9 Aug 2021 16:07:41 -0700 Subject: [PATCH] HPACK Table --> C++ (#26851) * Buffer HPACK parsing until the end of a header boundary HTTP2 headers are sent in (potentially) many frames, but all must be sent sequentially with no traffic intervening. This was not clear when I wrote the HPACK parser, and still indeed quite contentious on the HTTP2 mailing lists. Now that matter is well settled (years ago!) take advantage of the fact by delaying parsing until all bytes are available. A future change will leverage this to avoid having to store and verify partial parse state, completely eliminating indirect calls within the parser. * maybe fixes * xx * fix boundary detection * clang-format * Revert "xx" This reverts commit 258d712ed3aa6a5cdfd4eae16ab8049b95a96655. * fix tests * add missed check * fixes * fix * update tests * fix benchmark * properly unref * optimize final slice refcounting * cleanup bm_chttp2_hpack * start * new parser progress * refinement * get it compiling * bug-fix * build files * clang-tidy * fixes * fixes * fixes * fix-leaks * clang-tidy * comments * fix merge error * Revert "Buffer HPACK parsing until the end of a header boundary (#26700)" This reverts commit 8bab3e4bf470f73104dc384af7007e6012654ef0. * streaming hpack parser start * streaming parser * clang-format * Rework HPackTable into C++ * clang-tidy * fix merge * actually set the size of the entries array * better --- BUILD | 1 + CMakeLists.txt | 10 +- build_autogenerated.yaml | 6 + gRPC-C++.podspec | 5 + gRPC-Core.podspec | 5 + grpc.gemspec | 2 + grpc.gyp | 2 + package.xml | 2 + .../chttp2/transport/hpack_encoder.cc | 7 +- .../chttp2/transport/hpack_parser.cc | 1813 +++++++---------- .../transport/chttp2/transport/hpack_parser.h | 215 +- .../transport/chttp2/transport/hpack_table.cc | 201 +- .../transport/chttp2/transport/hpack_table.h | 198 +- .../ext/transport/chttp2/transport/parsing.cc | 3 +- .../chttp2/hpack_parser_fuzzer_test.cc | 3 +- .../transport/chttp2/hpack_parser_test.cc | 10 +- .../core/transport/chttp2/hpack_table_test.cc | 141 +- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 4 +- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + 20 files changed, 1052 insertions(+), 1580 deletions(-) diff --git a/BUILD b/BUILD index dca5c8cdee0..5a1b3e4475c 100644 --- a/BUILD +++ b/BUILD @@ -2615,6 +2615,7 @@ grpc_cc_library( "grpc_http_filters", "grpc_trace", "grpc_transport_chttp2_alpn", + "match", "popularity_count", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3669d280f3f..995161784d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2130,6 +2130,7 @@ target_link_libraries(grpc absl::inlined_vector absl::bind_front absl::statusor + absl::variant gpr ${_gRPC_SSL_LIBRARIES} address_sorting @@ -2686,6 +2687,7 @@ target_link_libraries(grpc_unsecure absl::inlined_vector absl::bind_front absl::statusor + absl::variant gpr address_sorting ) @@ -16172,7 +16174,7 @@ generate_pkgconfig( "gRPC" "high performance general RPC framework" "${gRPC_CORE_VERSION}" - "gpr openssl absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "gpr openssl absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc -laddress_sorting -lre2 -lupb -lcares -lz" "" "grpc.pc") @@ -16182,7 +16184,7 @@ generate_pkgconfig( "gRPC unsecure" "high performance general RPC framework without SSL" "${gRPC_CORE_VERSION}" - "gpr absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "gpr absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc_unsecure" "" "grpc_unsecure.pc") @@ -16192,7 +16194,7 @@ generate_pkgconfig( "gRPC++" "C++ wrapper for gRPC" "${gRPC_CPP_VERSION}" - "grpc absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "grpc absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc++" "" "grpc++.pc") @@ -16202,7 +16204,7 @@ generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" "${gRPC_CPP_VERSION}" - "grpc_unsecure absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "grpc_unsecure absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc++_unsecure" "" "grpc++_unsecure.pc") diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index a1b06503880..0d693f72850 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -744,7 +744,9 @@ libs: - src/core/lib/event_engine/sockaddr.h - src/core/lib/gprpp/atomic.h - src/core/lib/gprpp/dual_ref_counted.h + - src/core/lib/gprpp/match.h - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/overload.h - src/core/lib/gprpp/ref_counted.h - src/core/lib/gprpp/ref_counted_ptr.h - src/core/lib/http/format_request.h @@ -1515,6 +1517,7 @@ libs: - absl/container:inlined_vector - absl/functional:bind_front - absl/status:statusor + - absl/types:variant - gpr - libssl - address_sorting @@ -1781,7 +1784,9 @@ libs: - src/core/lib/event_engine/sockaddr.h - src/core/lib/gprpp/atomic.h - src/core/lib/gprpp/dual_ref_counted.h + - src/core/lib/gprpp/match.h - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/overload.h - src/core/lib/gprpp/ref_counted.h - src/core/lib/gprpp/ref_counted_ptr.h - src/core/lib/http/format_request.h @@ -2197,6 +2202,7 @@ libs: - absl/container:inlined_vector - absl/functional:bind_front - absl/status:statusor + - absl/types:variant - gpr - address_sorting baselib: true diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index dc163db12d1..e45c71fc80b 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -202,6 +202,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/synchronization/synchronization', abseil_version ss.dependency 'abseil/time/time', abseil_version ss.dependency 'abseil/types/optional', abseil_version + ss.dependency 'abseil/types/variant', abseil_version ss.source_files = 'src/core/ext/filters/client_channel/backend_metric.h', 'src/core/ext/filters/client_channel/backup_poller.h', @@ -557,9 +558,11 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/global_config_generic.h', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', @@ -1221,9 +1224,11 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/global_config_generic.h', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index fd06be02c1a..48dc321ef19 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -192,6 +192,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/synchronization/synchronization', abseil_version ss.dependency 'abseil/time/time', abseil_version ss.dependency 'abseil/types/optional', abseil_version + ss.dependency 'abseil/types/variant', abseil_version ss.compiler_flags = '-DBORINGSSL_PREFIX=GRPC -Wno-unreachable-code -Wno-shorten-64-to-32' ss.source_files = 'src/core/ext/filters/census/grpc_context.cc', @@ -933,10 +934,12 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/host_port.cc', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.cc', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', @@ -1809,9 +1812,11 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/global_config_generic.h', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', diff --git a/grpc.gemspec b/grpc.gemspec index 41cde032faf..38f5b353629 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -847,10 +847,12 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/gprpp/host_port.cc ) s.files += %w( src/core/lib/gprpp/host_port.h ) s.files += %w( src/core/lib/gprpp/manual_constructor.h ) + s.files += %w( src/core/lib/gprpp/match.h ) s.files += %w( src/core/lib/gprpp/memory.h ) s.files += %w( src/core/lib/gprpp/mpscq.cc ) s.files += %w( src/core/lib/gprpp/mpscq.h ) s.files += %w( src/core/lib/gprpp/orphanable.h ) + s.files += %w( src/core/lib/gprpp/overload.h ) s.files += %w( src/core/lib/gprpp/ref_counted.h ) s.files += %w( src/core/lib/gprpp/ref_counted_ptr.h ) s.files += %w( src/core/lib/gprpp/stat.h ) diff --git a/grpc.gyp b/grpc.gyp index 9bcb94d99cc..e131d0c6b72 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -474,6 +474,7 @@ 'absl/container:inlined_vector', 'absl/functional:bind_front', 'absl/status:statusor', + 'absl/types:variant', 'gpr', 'address_sorting', ], @@ -1133,6 +1134,7 @@ 'absl/container:inlined_vector', 'absl/functional:bind_front', 'absl/status:statusor', + 'absl/types:variant', 'gpr', 'address_sorting', ], diff --git a/package.xml b/package.xml index f9ae3a95007..6ccfa37bc40 100644 --- a/package.xml +++ b/package.xml @@ -827,10 +827,12 @@ + + diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc index 932415436d7..04f289dfbb2 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc @@ -586,7 +586,7 @@ static void GPR_ATTRIBUTE_NOINLINE hpack_enc_log(grpc_mdelem elem) { } static uint32_t dynidx(grpc_chttp2_hpack_compressor* c, uint32_t elem_index) { - return 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY + c->tail_remote_index + + return 1 + grpc_core::HPackTable::kLastStaticEntry + c->tail_remote_index + c->table_elems - elem_index; } @@ -833,7 +833,7 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c, if (is_static && (static_index = reinterpret_cast(GRPC_MDELEM_DATA(md)) - ->StaticIndex()) < GRPC_CHTTP2_LAST_STATIC_ENTRY) { + ->StaticIndex()) < grpc_core::HPackTable::kLastStaticEntry) { emit_indexed(c, static_cast(static_index + 1), &st); } else { hpack_enc(c, md, &st); @@ -847,7 +847,8 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c, if (is_static && (static_index = reinterpret_cast( GRPC_MDELEM_DATA(l->md)) - ->StaticIndex()) < GRPC_CHTTP2_LAST_STATIC_ENTRY) { + ->StaticIndex()) < + grpc_core::HPackTable::kLastStaticEntry) { emit_indexed(c, static_cast(static_index + 1), &st); } else { hpack_enc(c, l->md, &st); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.cc b/src/core/ext/transport/chttp2/transport/hpack_parser.cc index 3d6e1aa346b..7f14396b684 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.cc @@ -34,6 +34,7 @@ #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" #include "src/core/lib/debug/stats.h" #include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/match.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -52,20 +53,6 @@ namespace grpc_core { DebugOnlyTraceFlag grpc_trace_chttp2_hpack_parser(false, "chttp2_hpack_parser"); -/* How parsing works: - - The parser object keeps track of a function pointer which represents the - current parse state. - - Each time new bytes are presented, we call into the current state, which - recursively parses until all bytes in the given chunk are exhausted. - - The parse state that terminates then saves its function pointer to be the - current state so that it can resume when more bytes are available. - - It's expected that most optimizing compilers will turn this code into - a set of indirect jumps, and so not waste stack space. */ - /* state table for huffman decoding: given a state, gives an index/16 into next_sub_tbl. Taking that index and adding the value of the nibble being considered returns the next state. @@ -464,1121 +451,875 @@ struct Base64InverseTable { static GRPC_HPACK_CONSTEXPR_VALUE Base64InverseTable kBase64InverseTable; } // namespace -void HPackParser::FinishFrame() { - sink_ = Sink(); - dynamic_table_updates_allowed_ = 2; -} - -void GPR_ATTRIBUTE_NOINLINE HPackParser::LogHeader(grpc_mdelem md) { - char* k = grpc_slice_to_c_string(GRPC_MDKEY(md)); - char* v = nullptr; - if (grpc_is_binary_header_internal(GRPC_MDKEY(md))) { - v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX); - } else { - v = grpc_slice_to_c_string(GRPC_MDVALUE(md)); +// Input tracks the current byte through the input data and provides it +// via a simple stream interface. +class HPackParser::Input { + public: + Input(grpc_slice_refcount* current_slice_refcount, const uint8_t* begin, + const uint8_t* end) + : current_slice_refcount_(current_slice_refcount), + begin_(begin), + end_(end), + frontier_(begin) {} + + // If input is backed by a slice, retrieve its refcount. If not, return + // nullptr. + grpc_slice_refcount* slice_refcount() { return current_slice_refcount_; } + + // Have we reached the end of input? + bool end_of_stream() const { return begin_ == end_; } + // How many bytes until end of input + size_t remaining() const { return end_ - begin_; } + // Current position, as a pointer + const uint8_t* cur_ptr() const { return begin_; } + // End position, as a pointer + const uint8_t* end_ptr() const { return end_; } + // Move read position forward by n, unchecked + void Advance(size_t n) { begin_ += n; } + + // Retrieve the current character, or nullopt if end of stream + // Do not advance + absl::optional peek() const { + if (end_of_stream()) { + return {}; + } + return *begin_; } - gpr_log( - GPR_INFO, - "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d", - k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md), - grpc_slice_is_interned(GRPC_MDKEY(md)), - grpc_slice_is_interned(GRPC_MDVALUE(md))); - gpr_free(k); - gpr_free(v); -} -/* emission helpers */ -template -grpc_error_handle HPackParser::FinishHeader(grpc_mdelem md) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { - LogHeader(md); - } - if (action == TableAction::kAddToTable) { - GPR_DEBUG_ASSERT(GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_INTERNED || - GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC); - grpc_error_handle err = grpc_chttp2_hptbl_add(&table_, md); - if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) return err; + // Retrieve and advance past the current character, or return nullopt if end + // of stream + absl::optional Next() { + if (end_of_stream()) { + return UnexpectedEOF(absl::optional()); + } + return *begin_++; } - return sink_(md); -} -UnmanagedMemorySlice HPackParser::String::TakeExtern() { - UnmanagedMemorySlice s; - if (!copied_) { - GPR_DEBUG_ASSERT(!grpc_slice_is_interned(data_.referenced)); - s = static_cast(data_.referenced); - copied_ = true; - data_.referenced = UnmanagedMemorySlice(); - } else { - s = UnmanagedMemorySlice(data_.copied.str, data_.copied.length); + // Helper to parse a varint delta on top of value, return nullopt on failure + // (setting error) + absl::optional ParseVarint(uint32_t value) { + // TODO(ctiller): break out a variant of this when we know there are at + // least 5 bytes in input_ + auto cur = Next(); + if (!cur) return {}; + value += *cur & 0x7f; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + value += (*cur & 0x7f) << 7; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + value += (*cur & 0x7f) << 14; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + value += (*cur & 0x7f) << 21; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + uint32_t c = (*cur) & 0x7f; + // We might overflow here, so we need to be a little careful about the + // addition + if (c > 0xf) return ParseVarintOutOfRange(value, *cur); + const uint32_t add = c << 28; + if (add > 0xffffffffu - value) { + return ParseVarintOutOfRange(value, *cur); + } + value += add; + if ((*cur & 0x80) == 0) return value; + + // Spec weirdness: we can add an infinite stream of 0x80 at the end of a + // varint and still end up with a correctly encoded varint. + do { + cur = Next(); + if (!cur.has_value()) return {}; + } while (*cur == 0x80); + + // BUT... the last byte needs to be 0x00 or we'll overflow dramatically! + if (*cur == 0) return value; + return ParseVarintOutOfRange(value, *cur); } - data_.copied.length = 0; - return s; -} -ManagedMemorySlice HPackParser::String::TakeIntern() { - ManagedMemorySlice s; - if (!copied_) { - s = ManagedMemorySlice(&data_.referenced); - grpc_slice_unref_internal(data_.referenced); - copied_ = true; - data_.referenced = grpc_empty_slice(); - } else { - s = ManagedMemorySlice(data_.copied.str, data_.copied.length); + // Prefix for a string + struct StringPrefix { + // Number of bytes in input for string + uint32_t length; + // Is it huffman compressed + bool huff; + }; + + // Parse a string prefix + absl::optional ParseStringPrefix() { + auto cur = Next(); + if (!cur.has_value()) return {}; + // Huffman if the top bit is 1 + const bool huff = (*cur & 0x80) != 0; + // String length + uint32_t strlen = (*cur & 0x7f); + if (strlen == 0x7f) { + // all ones ==> varint string length + auto v = ParseVarint(0x7f); + if (!v.has_value()) return {}; + strlen = *v; + } + return StringPrefix{strlen, huff}; } - data_.copied.length = 0; - return s; -} -grpc_error_handle HPackParser::parse_next(const uint8_t* cur, - const uint8_t* end) { - state_ = *next_state_++; - return (this->*state_)(cur, end); -} + // Check if we saw an EOF.. must be verified before looking at TakeError + bool eof_error() const { return eof_error_; } -/* begin parsing a header: all functionality is encoded into lookup tables - above */ -grpc_error_handle HPackParser::parse_begin(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_begin; - return GRPC_ERROR_NONE; + // Extract the parse error, leaving the current error as NONE. + grpc_error_handle TakeError() { + grpc_error_handle out = error_; + error_ = GRPC_ERROR_NONE; + return out; } - switch (*cur >> 4) { - // Literal header not indexed. - // First byte format: 0000xxxx - // Where xxxx: - // 0000 - literal key - // 1111 - indexed key, varint encoded index - // other - indexed key, inline encoded index - case 0: - switch (*cur & 0xf) { - case 0: // literal key - return parse_lithdr_notidx_v(cur, end); - case 0xf: // varint encoded key index - return parse_lithdr_notidx_x(cur, end); - default: // inline encoded key index - return parse_lithdr_notidx(cur, end); - } - // Literal header never indexed. - // First byte format: 0001xxxx - // Where xxxx: - // 0000 - literal key - // 1111 - indexed key, varint encoded index - // other - indexed key, inline encoded index - case 1: - switch (*cur & 0xf) { - case 0: // literal key - return parse_lithdr_nvridx_v(cur, end); - case 0xf: // varint encoded key index - return parse_lithdr_nvridx_x(cur, end); - default: // inline encoded key index - return parse_lithdr_nvridx(cur, end); - } - // Update max table size. - // First byte format: 001xxxxx - // Where xxxxx: - // 11111 - max size is varint encoded - // other - max size is stored inline - case 2: - // inline encoded max table size - return parse_max_tbl_size(cur, end); - case 3: - if (*cur == 0x3f) { - // varint encoded max table size - return parse_max_tbl_size_x(cur, end); - } else { - // inline encoded max table size - return parse_max_tbl_size(cur, end); - } - // Literal header with incremental indexing. - // First byte format: 01xxxxxx - // Where xxxxxx: - // 000000 - literal key - // 111111 - indexed key, varint encoded index - // other - indexed key, inline encoded index - case 4: - if (*cur == 0x40) { - // literal key - return parse_lithdr_incidx_v(cur, end); - } - ABSL_FALLTHROUGH_INTENDED; - case 5: - case 6: - // inline encoded key index - return parse_lithdr_incidx(cur, end); - case 7: - if (*cur == 0x7f) { - // varint encoded key index - return parse_lithdr_incidx_x(cur, end); - } else { - // inline encoded key index - return parse_lithdr_incidx(cur, end); - } - // Indexed Header Field Representation - // First byte format: 1xxxxxxx - // Where xxxxxxx: - // 0000000 - illegal - // 1111111 - varint encoded field index - // other - inline encoded field index - case 8: - if (*cur == 0x80) { - // illegal value. - return parse_illegal_op(cur, end); - } - ABSL_FALLTHROUGH_INTENDED; - case 9: - case 10: - case 11: - case 12: - case 13: - case 14: - // inline encoded field index - return parse_indexed_field(cur, end); - case 15: - if (*cur == 0xff) { - // varint encoded field index - return parse_indexed_field_x(cur, end); - } else { - // inline encoded field index - return parse_indexed_field(cur, end); - } + // Set the current error - allows the rest of the code not to need to pass + // around StatusOr<> which would be prohibitive here. + GPR_ATTRIBUTE_NOINLINE void SetError(grpc_error_handle error) { + if (error_ != GRPC_ERROR_NONE || eof_error_) { + GRPC_ERROR_UNREF(error); + return; + } + error_ = error; + begin_ = end_; } - GPR_UNREACHABLE_CODE(abort()); -} -/* stream dependency and prioritization data: we just skip it */ -grpc_error_handle HPackParser::parse_stream_weight(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_weight; - return GRPC_ERROR_NONE; + // If no error is set, set it to the value produced by error_factory. + // Return return_value unchanged. + template + GPR_ATTRIBUTE_NOINLINE T MaybeSetErrorAndReturn(F error_factory, + T return_value) { + if (error_ != GRPC_ERROR_NONE || eof_error_) return return_value; + error_ = error_factory(); + begin_ = end_; + return return_value; } - return (this->*after_prioritization_)(cur + 1, end); -} - -grpc_error_handle HPackParser::parse_stream_dep3(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep3; - return GRPC_ERROR_NONE; + // Set the error to an unexpected eof, and return result (code golfed as this + // is a common case) + template + T UnexpectedEOF(T return_value) { + if (error_ != GRPC_ERROR_NONE) return return_value; + eof_error_ = true; + return return_value; } - return parse_stream_weight(cur + 1, end); -} - -grpc_error_handle HPackParser::parse_stream_dep2(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep2; - return GRPC_ERROR_NONE; + // Update the frontier - signifies we've successfully parsed another element + void UpdateFrontier() { frontier_ = begin_; } + + // Get the frontier - for buffering should we fail due to eof + const uint8_t* frontier() const { return frontier_; } + + private: + // Helper to set the error to out of range for ParseVarint + absl::optional ParseVarintOutOfRange(uint32_t value, + uint8_t last_byte) { + return MaybeSetErrorAndReturn( + [value, last_byte] { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrFormat( + "integer overflow in hpack integer decoding: have 0x%08x, " + "got byte 0x%02x on byte 5", + value, last_byte) + .c_str()); + }, + absl::optional()); } - return parse_stream_dep3(cur + 1, end); -} + // Refcount if we are backed by a slice + grpc_slice_refcount* current_slice_refcount_; + // Current input point + const uint8_t* begin_; + // End of stream point + const uint8_t* const end_; + // Frontier denotes the first byte past successfully processed input + const uint8_t* frontier_; + // Current error + grpc_error_handle error_ = GRPC_ERROR_NONE; + // If the error was EOF, we flag it here.. + bool eof_error_ = false; +}; -grpc_error_handle HPackParser::parse_stream_dep1(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep1; - return GRPC_ERROR_NONE; +// Helper to parse a string and turn it into a slice with appropriate memory +// management characteristics +class HPackParser::String { + public: + // Helper to specify a string should be internalized + struct Intern {}; + // Helper to specify a string should be externalized + struct Extern {}; + + private: + // Forward declare take functions... we'll need them in the public interface + UnmanagedMemorySlice Take(Extern); + ManagedMemorySlice Take(Intern); + + public: + // If a String is a Slice then unref + ~String() { + if (auto* p = absl::get_if(&value_)) { + grpc_slice_unref_internal(*p); + } } - return parse_stream_dep2(cur + 1, end); -} - -grpc_error_handle HPackParser::parse_stream_dep0(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep0; - return GRPC_ERROR_NONE; + // Take the value and leave this empty + // Use Intern/Extern to choose memory management + template + auto Take() -> decltype(this->Take(T())) { + return Take(T()); } - return parse_stream_dep1(cur + 1, end); -} - -grpc_error_handle HPackParser::InvalidHPackIndexError() { - return grpc_error_set_int( - grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Invalid HPACK index received"), - GRPC_ERROR_INT_INDEX, static_cast(index_)), - GRPC_ERROR_INT_SIZE, static_cast(table_.num_ents)); -} - -/* emit an indexed field; jumps to begin the next field on completion */ -grpc_error_handle HPackParser::finish_indexed_field(const uint8_t* cur, - const uint8_t* end) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(&table_, index_); - if (GPR_UNLIKELY(GRPC_MDISNULL(md))) { - return InvalidHPackIndexError(); + String(const String&) = delete; + String& operator=(const String&) = delete; + String(String&& other) noexcept : value_(std::move(other.value_)) { + other.value_ = absl::Span(); } - GRPC_STATS_INC_HPACK_RECV_INDEXED(); - grpc_error_handle err = FinishHeader(md); - if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) return err; - return parse_begin(cur, end); -} - -/* parse an indexed field with index < 127 */ -grpc_error_handle HPackParser::parse_indexed_field(const uint8_t* cur, - const uint8_t* end) { - dynamic_table_updates_allowed_ = 0; - index_ = (*cur) & 0x7f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return finish_indexed_field(cur + 1, end); -} - -/* parse an indexed field with index >= 127 */ -grpc_error_handle HPackParser::parse_indexed_field_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = {&HPackParser::finish_indexed_field}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0x7f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* When finishing with a header, get the cached md element for this index. - This is set in parse_value_string(). We ensure (in debug mode) that the - cached metadata corresponds with the index we are examining. */ -grpc_mdelem HPackParser::GetPrecomputedMDForIndex() { - GPR_DEBUG_ASSERT(md_for_index_.payload != 0); - GPR_DEBUG_ASSERT(static_cast(index_) == precomputed_md_index_); - grpc_mdelem md = md_for_index_; - GPR_DEBUG_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ -#ifndef NDEBUG - precomputed_md_index_ = -1; -#endif - return md; -} - -static const grpc_core::ManagedMemorySlice& get_indexed_key(grpc_mdelem md) { - GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(md)); - return static_cast( - grpc_slice_ref_internal(GRPC_MDKEY(md))); -} - -/* finish a literal header with incremental indexing */ -grpc_error_handle HPackParser::finish_lithdr_incidx(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(); - grpc_mdelem md = GetPrecomputedMDForIndex(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(get_indexed_key(md), value_.TakeIntern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* finish a literal header with incremental indexing with no index */ -grpc_error_handle HPackParser::finish_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(key_.TakeIntern(), value_.TakeIntern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a literal header with incremental indexing; index < 63 */ -grpc_error_handle HPackParser::parse_lithdr_incidx(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_incidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = (*cur) & 0x3f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return parse_string_prefix(cur + 1, end); -} - -/* parse a literal header with incremental indexing; index >= 63 */ -grpc_error_handle HPackParser::parse_lithdr_incidx_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_incidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0x3f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* parse a literal header with incremental indexing; index = 0 */ -grpc_error_handle HPackParser::parse_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_key_string, &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_literal_key, - &HPackParser::finish_lithdr_incidx_v}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - return parse_string_prefix(cur + 1, end); -} - -/* finish a literal header without incremental indexing */ -grpc_error_handle HPackParser::finish_lithdr_notidx(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(); - grpc_mdelem md = GetPrecomputedMDForIndex(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(get_indexed_key(md), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* finish a literal header without incremental indexing with index = 0 */ -grpc_error_handle HPackParser::finish_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(key_.TakeIntern(), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a literal header without incremental indexing; index < 15 */ -grpc_error_handle HPackParser::parse_lithdr_notidx(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_notidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = (*cur) & 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return parse_string_prefix(cur + 1, end); -} - -/* parse a literal header without incremental indexing; index >= 15 */ -grpc_error_handle HPackParser::parse_lithdr_notidx_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_notidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* parse a literal header without incremental indexing; index == 0 */ -grpc_error_handle HPackParser::parse_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_key_string, &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_literal_key, - &HPackParser::finish_lithdr_notidx_v}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - return parse_string_prefix(cur + 1, end); -} - -/* finish a literal header that is never indexed */ -grpc_error_handle HPackParser::finish_lithdr_nvridx(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(); - grpc_mdelem md = GetPrecomputedMDForIndex(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(get_indexed_key(md), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* finish a literal header that is never indexed with an extra value */ -grpc_error_handle HPackParser::finish_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(key_.TakeIntern(), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a literal header that is never indexed; index < 15 */ -grpc_error_handle HPackParser::parse_lithdr_nvridx(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_nvridx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = (*cur) & 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return parse_string_prefix(cur + 1, end); -} - -/* parse a literal header that is never indexed; index >= 15 */ -grpc_error_handle HPackParser::parse_lithdr_nvridx_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_nvridx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* parse a literal header that is never indexed; index == 0 */ -grpc_error_handle HPackParser::parse_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_key_string, &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_literal_key, - &HPackParser::finish_lithdr_nvridx_v}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - return parse_string_prefix(cur + 1, end); -} - -/* finish parsing a max table size change */ -grpc_error_handle HPackParser::finish_max_tbl_size(const uint8_t* cur, - const uint8_t* end) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { - gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", index_); - } - grpc_error_handle err = - grpc_chttp2_hptbl_set_current_table_size(&table_, index_); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a max table size change, max size < 15 */ -grpc_error_handle HPackParser::parse_max_tbl_size(const uint8_t* cur, - const uint8_t* end) { - if (dynamic_table_updates_allowed_ == 0) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "More than two max table size changes in a single frame")); + String& operator=(String&& other) noexcept { + value_ = std::move(other.value_); + other.value_ = absl::Span(); + return *this; } - dynamic_table_updates_allowed_--; - index_ = (*cur) & 0x1f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return finish_max_tbl_size(cur + 1, end); -} - -/* parse a max table size change, max size >= 15 */ -grpc_error_handle HPackParser::parse_max_tbl_size_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = {&HPackParser::finish_max_tbl_size}; - if (dynamic_table_updates_allowed_ == 0) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "More than two max table size changes in a single frame")); - } - dynamic_table_updates_allowed_--; - next_state_ = and_then; - index_ = 0x1f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} -/* a parse error: jam the parse state into parse_error, and return error */ -grpc_error_handle HPackParser::parse_error(const uint8_t* /*cur*/, - const uint8_t* /*end*/, - grpc_error_handle err) { - GPR_ASSERT(err != GRPC_ERROR_NONE); - if (last_error_ == GRPC_ERROR_NONE) { - last_error_ = GRPC_ERROR_REF(err); + // Parse a non-binary string + static absl::optional Parse(Input* input) { + auto pfx = input->ParseStringPrefix(); + if (!pfx.has_value()) return {}; + if (pfx->huff) { + // Huffman coded + std::vector output; + auto v = ParseHuff(input, pfx->length, + [&output](uint8_t c) { output.push_back(c); }); + if (!v) return {}; + return String(std::move(output)); + } + return ParseUncompressed(input, pfx->length); } - state_ = &HPackParser::still_parse_error; - return err; -} - -grpc_error_handle HPackParser::still_parse_error(const uint8_t* /*cur*/, - const uint8_t* /*end*/) { - return GRPC_ERROR_REF(last_error_); -} -grpc_error_handle HPackParser::parse_illegal_op(const uint8_t* cur, - const uint8_t* end) { - GPR_ASSERT(cur != end); - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("Illegal hpack op code ", *cur).c_str()); - return parse_error(cur, end, err); -} - -/* parse the 1st byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value0(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value0; - return GRPC_ERROR_NONE; + // Parse a binary string + static absl::optional ParseBinary(Input* input) { + auto pfx = input->ParseStringPrefix(); + if (!pfx.has_value()) return {}; + if (!pfx->huff) { + if (pfx->length > 0 && input->peek() == 0) { + // 'true-binary' + input->Advance(1); + return ParseUncompressed(input, pfx->length - 1); + } + // Base64 encoded... pull out the string, then unbase64 it + auto base64 = ParseUncompressed(input, pfx->length); + if (!base64.has_value()) return {}; + return Unbase64(input, std::move(*base64)); + } else { + // Huffman encoded... + std::vector decompressed; + // State here says either we don't know if it's base64 or binary, or we do + // and what is it. + enum class State { kUnsure, kBinary, kBase64 }; + State state = State::kUnsure; + auto decompressed_ok = + ParseHuff(input, pfx->length, [&state, &decompressed](uint8_t c) { + if (state == State::kUnsure) { + // First byte... if it's zero it's binary + if (c == 0) { + // Save the type, and skip the zero + state = State::kBinary; + return; + } else { + // Flag base64, store this value + state = State::kBase64; + } + } + // Non-first byte, or base64 first byte + decompressed.push_back(c); + }); + if (!decompressed_ok) return {}; + switch (state) { + case State::kUnsure: + // No bytes, empty span + return String(absl::Span()); + case State::kBinary: + // Binary, we're done + return String(std::move(decompressed)); + case State::kBase64: + // Base64 - unpack it + return Unbase64(input, String(std::move(decompressed))); + } + GPR_UNREACHABLE_CODE(abort();); + } } - *parsing_.value += (*cur) & 0x7f; - - if ((*cur) & 0x80) { - return parse_value1(cur + 1, end); - } else { - return parse_next(cur + 1, end); + private: + void AppendBytes(const uint8_t* data, size_t length); + explicit String(std::vector v) : value_(std::move(v)) {} + explicit String(absl::Span v) : value_(v) {} + String(grpc_slice_refcount* r, const uint8_t* begin, const uint8_t* end) + : value_(MakeSlice(r, begin, end)) {} + + // Given a refcount and a byte range, make a slice + static grpc_slice MakeSlice(grpc_slice_refcount* r, const uint8_t* begin, + const uint8_t* end) { + grpc_slice out; + out.refcount = r; + r->Ref(); + out.data.refcounted.bytes = const_cast(begin); + out.data.refcounted.length = end - begin; + return out; } -} -/* parse the 2nd byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value1(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value1; - return GRPC_ERROR_NONE; + // Parse some huffman encoded bytes, using output(uint8_t b) to emit each + // decoded byte. + template + static bool ParseHuff(Input* input, uint32_t length, Out output) { + GRPC_STATS_INC_HPACK_RECV_HUFFMAN(); + int16_t state = 0; + // Parse one half byte... we leverage some lookup tables to keep the logic + // here really simple. + auto nibble = [&output, &state](uint8_t nibble) { + int16_t emit = emit_sub_tbl[16 * emit_tbl[state] + nibble]; + int16_t next = next_sub_tbl[16 * next_tbl[state] + nibble]; + if (emit != -1) { + if (emit >= 0 && emit < 256) { + output(static_cast(emit)); + } else { + assert(emit == 256); + } + } + state = next; + }; + // If there's insufficient bytes remaining, return now. + if (input->remaining() < length) { + return input->UnexpectedEOF(false); + } + // Grab the byte range, and iterate through it. + const uint8_t* p = input->cur_ptr(); + input->Advance(length); + for (uint32_t i = 0; i < length; i++) { + nibble(p[i] >> 4); + nibble(p[i] & 0xf); + } + return true; } - *parsing_.value += ((static_cast(*cur)) & 0x7f) << 7; - - if ((*cur) & 0x80) { - return parse_value2(cur + 1, end); - } else { - return parse_next(cur + 1, end); + // Parse some uncompressed string bytes. + static absl::optional ParseUncompressed(Input* input, + uint32_t length) { + GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(); + // Check there's enough bytes + if (input->remaining() < length) { + return input->UnexpectedEOF(absl::optional()); + } + auto* refcount = input->slice_refcount(); + auto* p = input->cur_ptr(); + input->Advance(length); + if (refcount != nullptr) { + return String(refcount, p, p + length); + } else { + return String(absl::Span(p, length)); + } } -} -/* parse the 3rd byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value2(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value2; - return GRPC_ERROR_NONE; + // Turn base64 encoded bytes into not base64 encoded bytes. + // Only takes input to set an error on failure. + static absl::optional Unbase64(Input* input, String s) { + auto v = Match( + s.value_, + [](const grpc_slice& slice) { + return Unbase64Loop(GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)); + }, + [](absl::Span span) { + return Unbase64Loop(span.begin(), span.end()); + }, + [](const std::vector& vec) { + return Unbase64Loop(vec.data(), vec.data() + vec.size()); + }); + if (!v.has_value()) { + return input->MaybeSetErrorAndReturn( + [] { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "illegal base64 encoding"); + }, + absl::optional()); + } + return String(std::move(*v)); } - *parsing_.value += ((static_cast(*cur)) & 0x7f) << 14; + // Main loop for Unbase64 + static absl::optional> Unbase64Loop(const uint8_t* cur, + const uint8_t* end) { + std::vector out; + out.reserve(3 * (end - cur) / 4 + 3); + + // Decode 4 bytes at a time while we can + while (end - cur >= 4) { + uint32_t bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + uint32_t buffer = bits << 18; + ++cur; - if ((*cur) & 0x80) { - return parse_value3(cur + 1, end); - } else { - return parse_next(cur + 1, end); - } -} + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 12; + ++cur; -/* parse the 4th byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value3(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value3; - return GRPC_ERROR_NONE; - } + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 6; + ++cur; - *parsing_.value += ((static_cast(*cur)) & 0x7f) << 21; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits; + ++cur; - if ((*cur) & 0x80) { - return parse_value4(cur + 1, end); - } else { - return parse_next(cur + 1, end); - } -} + out.insert(out.end(), {static_cast(buffer >> 16), + static_cast(buffer >> 8), + static_cast(buffer)}); + } + // Deal with the last 0, 1, 2, or 3 bytes. + switch (end - cur) { + case 0: + return out; + case 1: + return {}; + case 2: { + uint32_t bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + uint32_t buffer = bits << 18; -/* parse the 5th byte of a varint into parsing_.value - depending on the byte, we may overflow, and care must be taken */ -grpc_error_handle HPackParser::parse_value4(const uint8_t* cur, - const uint8_t* end) { - uint8_t c; - uint32_t cur_value; - uint32_t add_value; + ++cur; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 12; - if (cur == end) { - state_ = &HPackParser::parse_value4; - return GRPC_ERROR_NONE; - } + if (buffer & 0xffff) return {}; + out.push_back(static_cast(buffer >> 16)); + return out; + } + case 3: { + uint32_t bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + uint32_t buffer = bits << 18; - c = (*cur) & 0x7f; - if (c > 0xf) { - goto error; - } + ++cur; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 12; - cur_value = *parsing_.value; - add_value = (static_cast(c)) << 28; - if (add_value > 0xffffffffu - cur_value) { - goto error; - } + ++cur; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 6; - *parsing_.value = cur_value + add_value; + ++cur; + if (buffer & 0xff) return {}; + out.push_back(static_cast(buffer >> 16)); + out.push_back(static_cast(buffer >> 8)); + return out; + } + } - if ((*cur) & 0x80) { - return parse_value5up(cur + 1, end); - } else { - return parse_next(cur + 1, end); + GPR_UNREACHABLE_CODE(return out;); } -error: - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat( - "integer overflow in hpack integer decoding: have 0x%08x, " - "got byte 0x%02x on byte 5", - *parsing_.value, *cur) - .c_str()); - return parse_error(cur, end, err); -} + absl::variant, std::vector> + value_; +}; -/* parse any trailing bytes in a varint: it's possible to append an arbitrary - number of 0x80's and not affect the value - a zero will terminate - and - anything else will overflow */ -grpc_error_handle HPackParser::parse_value5up(const uint8_t* cur, - const uint8_t* end) { - while (cur != end && *cur == 0x80) { - ++cur; +// Parser parses one frame + continuations worth of headers. +class HPackParser::Parser { + public: + Parser(Input* input, HPackParser::Sink* sink, HPackTable* table, + uint8_t* dynamic_table_updates_allowed) + : input_(input), + sink_(sink), + table_(table), + dynamic_table_updates_allowed_(dynamic_table_updates_allowed) {} + + // Skip any priority bits, or return false on failure + bool SkipPriority() { + if (input_->remaining() < 5) return input_->UnexpectedEOF(false); + input_->Advance(5); + return true; } - if (cur == end) { - state_ = &HPackParser::parse_value5up; - return GRPC_ERROR_NONE; + bool Parse() { + auto cur = *input_->Next(); + switch (cur >> 4) { + // Literal header not indexed - First byte format: 0000xxxx + // Literal header never indexed - First byte format: 0001xxxx + // Where xxxx: + // 0000 - literal key + // 1111 - indexed key, varint encoded index + // other - indexed key, inline encoded index + case 0: + case 1: + switch (cur & 0xf) { + case 0: // literal key + return FinishHeader( + ParseLiteralKey()); + case 0xf: // varint encoded key index + return FinishHeader( + ParseVarIdxKey(0xf)); + default: // inline encoded key index + return FinishHeader( + ParseIdxKey(cur & 0xf)); + } + // Update max table size. + // First byte format: 001xxxxx + // Where xxxxx: + // 11111 - max size is varint encoded + // other - max size is stored inline + case 2: + // inline encoded max table size + return FinishMaxTableSize(cur & 0x1f); + case 3: + if (cur == 0x3f) { + // varint encoded max table size + return FinishMaxTableSize(input_->ParseVarint(0x1f)); + } else { + // inline encoded max table size + return FinishMaxTableSize(cur & 0x1f); + } + // Literal header with incremental indexing. + // First byte format: 01xxxxxx + // Where xxxxxx: + // 000000 - literal key + // 111111 - indexed key, varint encoded index + // other - indexed key, inline encoded index + case 4: + if (cur == 0x40) { + // literal key + return FinishHeader( + ParseLiteralKey()); + } + ABSL_FALLTHROUGH_INTENDED; + case 5: + case 6: + // inline encoded key index + return FinishHeader( + ParseIdxKey(cur & 0x3f)); + case 7: + if (cur == 0x7f) { + // varint encoded key index + return FinishHeader( + ParseVarIdxKey(0x3f)); + } else { + // inline encoded key index + return FinishHeader( + ParseIdxKey(cur & 0x3f)); + } + // Indexed Header Field Representation + // First byte format: 1xxxxxxx + // Where xxxxxxx: + // 0000000 - illegal + // 1111111 - varint encoded field index + // other - inline encoded field index + case 8: + if (cur == 0x80) { + // illegal value. + return input_->MaybeSetErrorAndReturn( + [] { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Illegal hpack op code"); + }, + false); + } + ABSL_FALLTHROUGH_INTENDED; + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + // inline encoded field index + return FinishIndexed(cur & 0x7f); + case 15: + if (cur == 0xff) { + // varint encoded field index + return FinishIndexed(input_->ParseVarint(0x7f)); + } else { + // inline encoded field index + return FinishIndexed(cur & 0x7f); + } + } + GPR_UNREACHABLE_CODE(abort()); } - if (*cur == 0) { - return parse_next(cur + 1, end); + private: + void GPR_ATTRIBUTE_NOINLINE LogHeader(grpc_mdelem md) { + char* k = grpc_slice_to_c_string(GRPC_MDKEY(md)); + char* v = nullptr; + if (grpc_is_binary_header_internal(GRPC_MDKEY(md))) { + v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX); + } else { + v = grpc_slice_to_c_string(GRPC_MDVALUE(md)); + } + gpr_log( + GPR_INFO, + "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d", + k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md), + grpc_slice_is_interned(GRPC_MDKEY(md)), + grpc_slice_is_interned(GRPC_MDVALUE(md))); + gpr_free(k); + gpr_free(v); } - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat( - "integer overflow in hpack integer decoding: have 0x%08x, " - "got byte 0x%02x sometime after byte 5", - *parsing_.value, *cur) - .c_str()); - return parse_error(cur, end, err); -} - -/* parse a string prefix */ -grpc_error_handle HPackParser::parse_string_prefix(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_string_prefix; - return GRPC_ERROR_NONE; + // During FinishHeader, how should the header be treated in the hpack table + enum class TableAction { + // Add to the table + kAddToTable, + // Do not add to the table + kOmitFromTable, + }; + + template + bool FinishHeader(grpc_mdelem md) { + // Allow higher code to just pass in failures ... simplifies things a bit. + if (GRPC_MDISNULL(md)) return false; + // Log if desired + if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { + LogHeader(md); + } + // Add to the hpack table if needed + if (action == TableAction::kAddToTable) { + GPR_DEBUG_ASSERT(GRPC_MDELEM_STORAGE(md) == + GRPC_MDELEM_STORAGE_INTERNED || + GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC); + grpc_error_handle err = table_->Add(md); + if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) { + input_->SetError(err); + return false; + }; + } + // Pass up to the transport + grpc_error_handle err = (*sink_)(md); + if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) { + input_->SetError(err); + return false; + } + return true; } - strlen_ = (*cur) & 0x7f; - huff_ = (*cur) >> 7; - if (strlen_ == 0x7f) { - parsing_.value = &strlen_; - return parse_value0(cur + 1, end); - } else { - return parse_next(cur + 1, end); + // Parse a string encoded key and a string encoded value + template + grpc_mdelem ParseLiteralKey() { + auto key = String::Parse(input_); + if (!key.has_value()) return GRPC_MDNULL; + auto key_slice = key->Take(); + auto value = ParseValueString(key_slice); + if (GPR_UNLIKELY(!value.has_value())) { + grpc_slice_unref_internal(key_slice); + return GRPC_MDNULL; + } + return grpc_mdelem_from_slices(key_slice, value->Take()); } -} -/* append some bytes to a string */ -void HPackParser::String::AppendBytes(const uint8_t* data, size_t length) { - if (length == 0) return; - if (length + data_.copied.length > data_.copied.capacity) { - GPR_ASSERT(data_.copied.length + length <= UINT32_MAX); - data_.copied.capacity = static_cast(data_.copied.length + length); - data_.copied.str = static_cast( - gpr_realloc(data_.copied.str, data_.copied.capacity)); + // Parse an index encoded key and a string encoded value + template + grpc_mdelem ParseIdxKey(uint32_t index) { + auto elem = table_->Peek(index); + if (GPR_UNLIKELY(GRPC_MDISNULL(elem))) { + return InvalidHPackIndexError(index, elem); + } + GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(elem)); + auto value = ParseValueString(GRPC_MDKEY(elem)); + if (GPR_UNLIKELY(!value.has_value())) return GRPC_MDNULL; + return grpc_mdelem_from_slices( + static_cast( + grpc_slice_ref_internal(GRPC_MDKEY(elem))), + value->Take()); } - memcpy(data_.copied.str + data_.copied.length, data, length); - GPR_ASSERT(length <= UINT32_MAX - data_.copied.length); - data_.copied.length += static_cast(length); -} -grpc_error_handle HPackParser::AppendString(const uint8_t* cur, - const uint8_t* end) { - String* str = parsing_.str; - uint32_t bits; - uint8_t decoded[3]; - switch (binary_) { - case BinaryState::kNotBinary: - str->AppendBytes(cur, static_cast(end - cur)); - return GRPC_ERROR_NONE; - case BinaryState::kBinaryBegin: - if (cur == end) { - binary_ = BinaryState::kBinaryBegin; - return GRPC_ERROR_NONE; - } - if (*cur == 0) { - /* 'true-binary' case */ - ++cur; - binary_ = BinaryState::kNotBinary; - GRPC_STATS_INC_HPACK_RECV_BINARY(); - str->AppendBytes(cur, static_cast(end - cur)); - return GRPC_ERROR_NONE; - } - GRPC_STATS_INC_HPACK_RECV_BINARY_BASE64(); - b64_byte0: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte0: - if (cur == end) { - binary_ = BinaryState::kBase64Byte0; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte0; - } - base64_buffer_ = bits << 18; - b64_byte1: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte1: - if (cur == end) { - binary_ = BinaryState::kBase64Byte1; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte1; - } - base64_buffer_ |= bits << 12; - b64_byte2: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte2: - if (cur == end) { - binary_ = BinaryState::kBase64Byte2; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte2; - } - base64_buffer_ |= bits << 6; - b64_byte3: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte3: - if (cur == end) { - binary_ = BinaryState::kBase64Byte3; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte3; - } - base64_buffer_ |= bits; - bits = base64_buffer_; - decoded[0] = static_cast(bits >> 16); - decoded[1] = static_cast(bits >> 8); - decoded[2] = static_cast(bits); - str->AppendBytes(decoded, 3); - goto b64_byte0; + // Parse a varint index encoded key and a string encoded value + template + grpc_mdelem ParseVarIdxKey(uint32_t offset) { + auto index = input_->ParseVarint(offset); + if (GPR_UNLIKELY(!index.has_value())) return GRPC_MDNULL; + return ParseIdxKey(*index); } - GPR_UNREACHABLE_CODE(return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Should never reach here"))); -} -grpc_error_handle HPackParser::finish_str(const uint8_t* cur, - const uint8_t* end) { - uint8_t decoded[2]; - uint32_t bits; - String* str = parsing_.str; - switch (binary_) { - case BinaryState::kNotBinary: - break; - case BinaryState::kBinaryBegin: - break; - case BinaryState::kBase64Byte0: - break; - case BinaryState::kBase64Byte1: - return parse_error(cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "illegal base64 encoding")); /* illegal encoding */ - case BinaryState::kBase64Byte2: - bits = base64_buffer_; - if (bits & 0xffff) { - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat("trailing bits in base64 encoding: 0x%04x", - bits & 0xffff) - .c_str()); - return parse_error(cur, end, err); - } - decoded[0] = static_cast(bits >> 16); - str->AppendBytes(decoded, 1); - break; - case BinaryState::kBase64Byte3: - bits = base64_buffer_; - if (bits & 0xff) { - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat("trailing bits in base64 encoding: 0x%02x", - bits & 0xff) - .c_str()); - return parse_error(cur, end, err); - } - decoded[0] = static_cast(bits >> 16); - decoded[1] = static_cast(bits >> 8); - str->AppendBytes(decoded, 2); - break; - } - return GRPC_ERROR_NONE; -} - -/* decode a nibble from a huffman encoded stream */ -grpc_error_handle HPackParser::AppendHuffNibble(uint8_t nibble) { - int16_t emit = emit_sub_tbl[16 * emit_tbl[huff_state_] + nibble]; - int16_t next = next_sub_tbl[16 * next_tbl[huff_state_] + nibble]; - if (emit != -1) { - if (emit >= 0 && emit < 256) { - uint8_t c = static_cast(emit); - grpc_error_handle err = AppendString(&c, (&c) + 1); - if (err != GRPC_ERROR_NONE) return err; + // Parse a string, figuring out if it's binary or not by the key name. + template + absl::optional ParseValueString(const SliceType& key) { + if (grpc_is_refcounted_slice_binary_header(key)) { + return String::ParseBinary(input_); } else { - assert(emit == 256); + return String::Parse(input_); } } - huff_state_ = next; - return GRPC_ERROR_NONE; -} -/* decode full bytes from a huffman encoded stream */ -grpc_error_handle HPackParser::AppendHuffBytes(const uint8_t* cur, - const uint8_t* end) { - for (; cur != end; ++cur) { - grpc_error_handle err = AppendHuffNibble(*cur >> 4); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - err = AppendHuffNibble(*cur & 0xf); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); + // Emit an indexed field + bool FinishIndexed(absl::optional index) { + *dynamic_table_updates_allowed_ = 0; + if (!index.has_value()) return false; + grpc_mdelem md = table_->Fetch(*index); + if (GPR_UNLIKELY(GRPC_MDISNULL(md))) { + return InvalidHPackIndexError(*index, false); + } + GRPC_STATS_INC_HPACK_RECV_INDEXED(); + return FinishHeader(md); } - return GRPC_ERROR_NONE; -} -/* decode some string bytes based on the current decoding mode - (huffman or not) */ -grpc_error_handle HPackParser::AppendStrBytes(const uint8_t* cur, - const uint8_t* end) { - if (huff_) { - return AppendHuffBytes(cur, end); - } else { - return AppendString(cur, end); + // finish parsing a max table size change + bool FinishMaxTableSize(absl::optional size) { + if (!size.has_value()) return false; + if (*dynamic_table_updates_allowed_ == 0) { + return input_->MaybeSetErrorAndReturn( + [] { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "More than two max table size changes in a single frame"); + }, + false); + } + (*dynamic_table_updates_allowed_)--; + if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { + gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", *size); + } + grpc_error_handle err = table_->SetCurrentTableSize(*size); + if (err != GRPC_ERROR_NONE) { + input_->SetError(err); + return false; + } + return true; } -} -/* parse a string - tries to do large chunks at a time */ -grpc_error_handle HPackParser::parse_string(const uint8_t* cur, - const uint8_t* end) { - size_t remaining = strlen_ - strgot_; - size_t given = static_cast(end - cur); - if (remaining <= given) { - grpc_error_handle err = AppendStrBytes(cur, cur + remaining); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - err = finish_str(cur + remaining, end); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_next(cur + remaining, end); - } else { - grpc_error_handle err = AppendStrBytes(cur, cur + given); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - GPR_ASSERT(given <= UINT32_MAX - strgot_); - strgot_ += static_cast(given); - state_ = &HPackParser::parse_string; - return GRPC_ERROR_NONE; + // Set an invalid hpack index error if no error has been set. Returns result + // unmodified. + template + R InvalidHPackIndexError(uint32_t index, R result) { + return input_->MaybeSetErrorAndReturn( + [this, index] { + return grpc_error_set_int( + grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Invalid HPACK index received"), + GRPC_ERROR_INT_INDEX, + static_cast(index)), + GRPC_ERROR_INT_SIZE, + static_cast(this->table_->num_entries())); + }, + result); } -} -/* begin parsing a string - performs setup, calls parse_string */ -grpc_error_handle HPackParser::begin_parse_string(const uint8_t* cur, - const uint8_t* end, - BinaryState binary, - HPackParser::String* str) { - if (!huff_ && binary == BinaryState::kNotBinary && - static_cast(end - cur) >= strlen_ && - current_slice_refcount_ != nullptr) { - GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(); - str->copied_ = false; - str->data_.referenced.refcount = current_slice_refcount_; - str->data_.referenced.data.refcounted.bytes = const_cast(cur); - str->data_.referenced.data.refcounted.length = strlen_; - grpc_slice_ref_internal(str->data_.referenced); - return parse_next(cur + strlen_, end); - } - strgot_ = 0; - str->copied_ = true; - str->data_.copied.length = 0; - parsing_.str = str; - huff_state_ = 0; - binary_ = binary; - switch (binary_) { - case BinaryState::kNotBinary: - if (huff_) { - GRPC_STATS_INC_HPACK_RECV_HUFFMAN(); - } else { - GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(); - } - break; - case BinaryState::kBinaryBegin: - /* stats incremented later: don't know true binary or not */ - break; - default: - abort(); - } - return parse_string(cur, end); -} + Input* input_; + HPackParser::Sink* sink_; + HPackTable* const table_; + uint8_t* dynamic_table_updates_allowed_; +}; -/* parse the key string */ -grpc_error_handle HPackParser::parse_key_string(const uint8_t* cur, - const uint8_t* end) { - return begin_parse_string(cur, end, BinaryState::kNotBinary, &key_); +UnmanagedMemorySlice HPackParser::String::Take(Extern) { + auto s = Match( + value_, + [](const grpc_slice& slice) { + GPR_DEBUG_ASSERT(!grpc_slice_is_interned(slice)); + return static_cast(slice); + }, + [](absl::Span span) { + return UnmanagedMemorySlice( + reinterpret_cast(const_cast(span.begin())), + span.size()); + }, + [](const std::vector& v) { + return UnmanagedMemorySlice(reinterpret_cast(v.data()), + v.size()); + }); + value_ = absl::Span(); + return s; } -/* check if a key represents a binary header or not */ - -bool HPackParser::IsBinaryLiteralHeader() { - /* We know that either argument here is a reference counter slice. - * 1. If it is a grpc_core::StaticSlice, the refcount is set to kNoopRefcount. - * 2. If it's key_.data.referenced, then key_.copied was set to false, - * which occurs in begin_parse_string() - where the refcount is set to - * current_slice_refcount_, which is not null. */ - return grpc_is_refcounted_slice_binary_header( - key_.copied_ ? grpc_core::ExternallyManagedSlice(key_.data_.copied.str, - key_.data_.copied.length) - : key_.data_.referenced); +ManagedMemorySlice HPackParser::String::Take(Intern) { + auto s = Match( + value_, + [](const grpc_slice& slice) { + ManagedMemorySlice s(&slice); + grpc_slice_unref_internal(slice); + return s; + }, + [](absl::Span span) { + return ManagedMemorySlice( + reinterpret_cast(const_cast(span.data())), + span.size()); + }, + [](const std::vector& v) { + return ManagedMemorySlice(reinterpret_cast(v.data()), + v.size()); + }); + value_ = absl::Span(); + return s; } -/* Cache the metadata for the given index during initial parsing. This avoids a - pointless recomputation of the metadata when finishing a header. We read the - cached value in get_precomputed_md_for_idx(). */ -void HPackParser::SetPrecomputedMDIndex(grpc_mdelem md) { - GPR_DEBUG_ASSERT(md_for_index_.payload == 0); - GPR_DEBUG_ASSERT(precomputed_md_index_ == -1); - md_for_index_ = md; -#ifndef NDEBUG - precomputed_md_index_ = index_; -#endif -} +/* PUBLIC INTERFACE */ -/* Determines if a metadata element key associated with the current parser index - is a binary indexed header during string parsing. We'll need to revisit this - metadata when we're done parsing, so we cache the metadata for this index - here using set_precomputed_md_idx(). */ -grpc_error_handle HPackParser::IsBinaryIndexedHeader(bool* is) { - grpc_mdelem elem = grpc_chttp2_hptbl_lookup(&table_, index_); - if (GPR_UNLIKELY(GRPC_MDISNULL(elem))) { - return InvalidHPackIndexError(); - } - /* We know that GRPC_MDKEY(elem) points to a reference counted slice since: - * 1. elem was a result of grpc_chttp2_hptbl_lookup - * 2. An item in this table is either static (see entries with - * index < GRPC_CHTTP2_LAST_STATIC_ENTRY or added via - * grpc_chttp2_hptbl_add). - * 3. If added via grpc_chttp2_hptbl_add, the entry is either static or - * interned. - * 4. Both static and interned element slices have non-null refcounts. */ - *is = grpc_is_refcounted_slice_binary_header(GRPC_MDKEY(elem)); - SetPrecomputedMDIndex(elem); - return GRPC_ERROR_NONE; -} +HPackParser::HPackParser() = default; -/* parse the value string */ -grpc_error_handle HPackParser::parse_value_string(const uint8_t* cur, - const uint8_t* end, - bool is_binary) { - return begin_parse_string( - cur, end, is_binary ? BinaryState::kBinaryBegin : BinaryState::kNotBinary, - &value_); -} +HPackParser::~HPackParser() = default; -grpc_error_handle HPackParser::parse_value_string_with_indexed_key( - const uint8_t* cur, const uint8_t* end) { - bool is_binary = false; - grpc_error_handle err = IsBinaryIndexedHeader(&is_binary); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_value_string(cur, end, is_binary); +void HPackParser::BeginFrame(Sink sink, Boundary boundary, Priority priority) { + sink_ = std::move(sink); + boundary_ = boundary; + priority_ = priority; + dynamic_table_updates_allowed_ = 2; } -grpc_error_handle HPackParser::parse_value_string_with_literal_key( - const uint8_t* cur, const uint8_t* end) { - return parse_value_string(cur, end, IsBinaryLiteralHeader()); +grpc_error_handle HPackParser::Parse(const grpc_slice& slice, bool is_last) { + if (GPR_UNLIKELY(!unparsed_bytes_.empty())) { + std::vector buffer = std::move(unparsed_bytes_); + buffer.insert(buffer.end(), GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)); + return ParseInput( + Input(nullptr, buffer.data(), buffer.data() + buffer.size()), is_last); + } + return ParseInput(Input(slice.refcount, GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)), + is_last); } -/* PUBLIC INTERFACE */ - -HPackParser::HPackParser() { - state_ = &HPackParser::parse_begin; - key_.data_.referenced = grpc_empty_slice(); - key_.data_.copied.str = nullptr; - key_.data_.copied.capacity = 0; - key_.data_.copied.length = 0; - value_.data_.referenced = grpc_empty_slice(); - value_.data_.copied.str = nullptr; - value_.data_.copied.capacity = 0; - value_.data_.copied.length = 0; - /* Cached metadata for the current index the parser is handling. This is set - to 0 initially, invalidated when the index changes, and invalidated when it - is read (by get_precomputed_md_for_idx()). It is set during string parsing, - by set_precomputed_md_idx() - which is called by parse_value_string(). - The goal here is to avoid recomputing the metadata for the index when - finishing with a header as well as the initial parse. */ - md_for_index_.payload = 0; -#ifndef NDEBUG - /* In debug mode, this ensures that the cached metadata we're reading is in - * fact correct for the index we are examining. */ - precomputed_md_index_ = -1; -#endif - dynamic_table_updates_allowed_ = 2; - last_error_ = GRPC_ERROR_NONE; +grpc_error_handle HPackParser::ParseInput(Input input, bool is_last) { + if (ParseInputInner(&input)) { + return GRPC_ERROR_NONE; + } + if (input.eof_error()) { + if (GPR_UNLIKELY(is_last && is_boundary())) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Incomplete header at the end of a header/continuation sequence"); + } + unparsed_bytes_ = std::vector(input.frontier(), input.end_ptr()); + return GRPC_ERROR_NONE; + } + return input.TakeError(); } -void HPackParser::BeginFrame(Sink sink, Boundary boundary, Priority priority) { - sink_ = std::move(sink); - boundary_ = boundary; - switch (priority) { - case Priority::Included: - after_prioritization_ = state_; - state_ = &HPackParser::parse_stream_dep0; - break; +bool HPackParser::ParseInputInner(Input* input) { + switch (priority_) { case Priority::None: break; + case Priority::Included: { + if (input->remaining() < 5) return input->UnexpectedEOF(false); + input->Advance(5); + input->UpdateFrontier(); + priority_ = Priority::None; + } } -} - -HPackParser::~HPackParser() { - grpc_chttp2_hptbl_destroy(&table_); - GRPC_ERROR_UNREF(last_error_); - grpc_slice_unref_internal(key_.data_.referenced); - grpc_slice_unref_internal(value_.data_.referenced); - gpr_free(key_.data_.copied.str); - gpr_free(value_.data_.copied.str); -} - -grpc_error_handle HPackParser::Parse(const grpc_slice& slice) { -/* max number of bytes to parse at a time... limits call stack depth on - * compilers without TCO */ -#define MAX_PARSE_LENGTH 1024 - current_slice_refcount_ = slice.refcount; - const uint8_t* start = GRPC_SLICE_START_PTR(slice); - const uint8_t* end = GRPC_SLICE_END_PTR(slice); - grpc_error_handle error = GRPC_ERROR_NONE; - while (start != end && error == GRPC_ERROR_NONE) { - const uint8_t* target = start + GPR_MIN(MAX_PARSE_LENGTH, end - start); - error = (this->*state_)(start, target); - start = target; + while (!input->end_of_stream()) { + if (GPR_UNLIKELY( + !Parser(input, &sink_, &table_, &dynamic_table_updates_allowed_) + .Parse())) { + return false; + } + input->UpdateFrontier(); } - current_slice_refcount_ = nullptr; - return error; + return true; } +void HPackParser::FinishFrame() { sink_ = Sink(); } + } // namespace grpc_core // TODO(ctiller): this serves as an eviction notice for the remainder of this @@ -1630,15 +1371,11 @@ grpc_error_handle grpc_chttp2_header_parser_parse(void* hpack_parser, if (s != nullptr) { s->stats.incoming.header_bytes += GRPC_SLICE_LENGTH(slice); } - grpc_error_handle error = parser->Parse(slice); + grpc_error_handle error = parser->Parse(slice, is_last != 0); if (error != GRPC_ERROR_NONE) { return error; } if (is_last) { - if (parser->is_boundary() && !parser->is_in_begin_state()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "end of header frame not aligned with a hpack record boundary"); - } /* need to check for null stream: this can occur if we receive an invalid stream id on a header */ if (s != nullptr) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h index 01fb833df1e..d6b6a49e03b 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.h +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h @@ -29,10 +29,25 @@ namespace grpc_core { +// Top level interface for parsing a sequence of header, continuation frames. class HPackParser { public: - enum class Boundary { None, EndOfHeaders, EndOfStream }; - enum class Priority { None, Included }; + // What kind of stream boundary is provided by this frame? + enum class Boundary : uint8_t { + // More continuations are expected + None, + // This marks the end of headers, so data frames should follow + EndOfHeaders, + // This marks the end of headers *and* the end of the stream + EndOfStream + }; + // What kind of priority is represented in the next frame + enum class Priority : uint8_t { + // No priority field + None, + // Yes there's a priority field + Included + }; // User specified structure called for each received header. using Sink = std::function; @@ -40,189 +55,53 @@ class HPackParser { HPackParser(); ~HPackParser(); + // Non-copyable/movable HPackParser(const HPackParser&) = delete; HPackParser& operator=(const HPackParser&) = delete; + // Begin parsing a new frame + // Sink receives each parsed header, void BeginFrame(Sink sink, Boundary boundary, Priority priority); + // Change the header sink mid parse void ResetSink(Sink sink) { sink_ = std::move(sink); } - grpc_error_handle Parse(const grpc_slice& slice); + // Parse one slice worth of data + grpc_error_handle Parse(const grpc_slice& slice, bool is_last); + // Reset state ready for the next BeginFrame void FinishFrame(); - grpc_chttp2_hptbl* hpack_table() { return &table_; } + // Retrieve the associated hpack table (for tests, debugging) + HPackTable* hpack_table() { return &table_; } + // Is the current frame a boundary of some sort bool is_boundary() const { return boundary_ != Boundary::None; } + // Is the current frame the end of a stream bool is_eof() const { return boundary_ == Boundary::EndOfStream; } - bool is_in_begin_state() const { return state_ == &HPackParser::parse_begin; } private: - enum class BinaryState { - kNotBinary, - kBinaryBegin, - kBase64Byte0, - kBase64Byte1, - kBase64Byte2, - kBase64Byte3, - }; - - struct String { - bool copied_; - struct { - grpc_slice referenced; - struct { - char* str; - uint32_t length; - uint32_t capacity; - } copied; - } data_; - - UnmanagedMemorySlice TakeExtern(); - ManagedMemorySlice TakeIntern(); - void AppendBytes(const uint8_t* data, size_t length); - }; - - using State = grpc_error_handle (HPackParser::*)(const uint8_t* beg, - const uint8_t* end); - - // Forward declarations for parsing states. - // These are keeping their old (C-style) names until a future refactor where - // they will be eliminated. - grpc_error_handle parse_next(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_begin(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_error(const uint8_t* cur, const uint8_t* end, - grpc_error_handle error); - grpc_error_handle still_parse_error(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_illegal_op(const uint8_t* cur, const uint8_t* end); - - grpc_error_handle parse_string_prefix(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_key_string(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value_string(const uint8_t* cur, const uint8_t* end, - bool is_binary); - grpc_error_handle parse_value_string_with_indexed_key(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_value_string_with_literal_key(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_stream_weight(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value0(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value1(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value2(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value3(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value4(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value5up(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep0(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep1(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep2(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep3(const uint8_t* cur, const uint8_t* end); - - grpc_error_handle parse_indexed_field(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_indexed_field_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_incidx(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_lithdr_incidx_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_notidx(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_lithdr_notidx_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_nvridx(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_lithdr_nvridx_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_max_tbl_size(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_max_tbl_size_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_string(const uint8_t* cur, const uint8_t* end); - grpc_error_handle begin_parse_string(const uint8_t* cur, const uint8_t* end, - BinaryState binary, String* str); - - grpc_error_handle finish_indexed_field(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_incidx(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_notidx(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_nvridx(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_max_tbl_size(const uint8_t* cur, const uint8_t* end); - grpc_error_handle finish_str(const uint8_t* cur, const uint8_t* end); - - enum class TableAction { - kAddToTable, - kOmitFromTable, - }; - - GPR_ATTRIBUTE_NOINLINE grpc_error_handle InvalidHPackIndexError(); - GPR_ATTRIBUTE_NOINLINE void LogHeader(grpc_mdelem md); - grpc_error_handle AddHeaderToTable(grpc_mdelem md); - template - grpc_error_handle FinishHeader(grpc_mdelem md); - - grpc_mdelem GetPrecomputedMDForIndex(); - void SetPrecomputedMDIndex(grpc_mdelem md); - bool IsBinaryLiteralHeader(); - grpc_error_handle IsBinaryIndexedHeader(bool* is); + // Helper classes: see implementation + class Parser; + class Input; + class String; - grpc_error_handle AppendString(const uint8_t* cur, const uint8_t* end); - grpc_error_handle AppendHuffNibble(uint8_t nibble); - grpc_error_handle AppendHuffBytes(const uint8_t* cur, const uint8_t* end); - grpc_error_handle AppendStrBytes(const uint8_t* cur, const uint8_t* end); + grpc_error_handle ParseInput(Input input, bool is_last); + bool ParseInputInner(Input* input); + // Callback per header received Sink sink_; - grpc_error_handle last_error_; - - // current parse state - or a function that implements it - State state_; - // future states dependent on the opening op code - const State* next_state_; - // what to do after skipping prioritization data - State after_prioritization_; - // the refcount of the slice that we're currently parsing - grpc_slice_refcount* current_slice_refcount_; - // the value we're currently parsing - union { - uint32_t* value; - String* str; - } parsing_; - // string parameters for each chunk - String key_; - String value_; - // parsed index - uint32_t index_; - // When we parse a value string, we determine the metadata element for a - // specific index, which we need again when we're finishing up with that - // header. To avoid calculating the metadata element for that index a second - // time at that stage, we cache (and invalidate) the element here. - grpc_mdelem md_for_index_; -#ifndef NDEBUG - int64_t precomputed_md_index_; -#endif - // length of source bytes for the currently parsing string - uint32_t strlen_; - // number of source bytes read for the currently parsing string - uint32_t strgot_; - // huffman decoding state - int16_t huff_state_; - // is the string being decoded binary? - BinaryState binary_; - // is the current string huffman encoded? - bool huff_; - // is a dynamic table update allowed? - uint8_t dynamic_table_updates_allowed_; - // set by higher layers, used by grpc_chttp2_header_parser_parse to signal - // it should append a metadata boundary at the end of frame + + // Bytes that could not be parsed last parsing round + std::vector unparsed_bytes_; + // Buffer kind of boundary + // TODO(ctiller): see if we can move this argument to Parse, and avoid + // buffering. Boundary boundary_; - uint32_t base64_buffer_; + // Buffer priority + // TODO(ctiller): see if we can move this argument to Parse, and avoid + // buffering. + Priority priority_; + uint8_t dynamic_table_updates_allowed_; // hpack table - grpc_chttp2_hptbl table_; + HPackTable table_; }; } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.cc b/src/core/ext/transport/chttp2/transport/hpack_table.cc index 696176b49d5..4e437fc982e 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_table.cc @@ -36,191 +36,132 @@ extern grpc_core::TraceFlag grpc_http_trace; -void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl* tbl) { - size_t i; - for (i = 0; i < tbl->num_ents; i++) { - GRPC_MDELEM_UNREF(tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]); - } - gpr_free(tbl->ents); - tbl->ents = nullptr; -} +namespace grpc_core { -template -static grpc_mdelem lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index) { - /* Not static - find the value in the list of valid entries */ - tbl_index -= (GRPC_CHTTP2_LAST_STATIC_ENTRY + 1); - if (tbl_index < tbl->num_ents) { - uint32_t offset = - (tbl->num_ents - 1u - tbl_index + tbl->first_ent) % tbl->cap_entries; - grpc_mdelem md = tbl->ents[offset]; - if (take_ref) { - GRPC_MDELEM_REF(md); - } - return md; - } - /* Invalid entry: return error */ - return GRPC_MDNULL; -} +using hpack_table_detail::EntriesForBytes; +using hpack_table_detail::kInlineEntries; -grpc_mdelem grpc_chttp2_hptbl_lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index) { - return lookup_dynamic_index(tbl, tbl_index); -} +HPackTable::HPackTable() : entries_(kInlineEntries) {} -grpc_mdelem grpc_chttp2_hptbl_lookup_ref_dynamic_index( - const grpc_chttp2_hptbl* tbl, uint32_t tbl_index) { - return lookup_dynamic_index(tbl, tbl_index); +HPackTable::~HPackTable() { + for (size_t i = 0; i < num_entries_; i++) { + GRPC_MDELEM_UNREF(entries_[(first_entry_ + i) % entries_.size()]); + } } /* Evict one element from the table */ -static void evict1(grpc_chttp2_hptbl* tbl) { - grpc_mdelem first_ent = tbl->ents[tbl->first_ent]; - size_t elem_bytes = GRPC_SLICE_LENGTH(GRPC_MDKEY(first_ent)) + - GRPC_SLICE_LENGTH(GRPC_MDVALUE(first_ent)) + - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; - GPR_ASSERT(elem_bytes <= tbl->mem_used); - tbl->mem_used -= static_cast(elem_bytes); - tbl->first_ent = ((tbl->first_ent + 1) % tbl->cap_entries); - tbl->num_ents--; - GRPC_MDELEM_UNREF(first_ent); +void HPackTable::EvictOne() { + grpc_mdelem first_entry = entries_[first_entry_]; + size_t elem_bytes = GRPC_SLICE_LENGTH(GRPC_MDKEY(first_entry)) + + GRPC_SLICE_LENGTH(GRPC_MDVALUE(first_entry)) + + kEntryOverhead; + GPR_ASSERT(elem_bytes <= mem_used_); + mem_used_ -= static_cast(elem_bytes); + first_entry_ = ((first_entry_ + 1) % entries_.size()); + num_entries_--; + GRPC_MDELEM_UNREF(first_entry); } -static void rebuild_ents(grpc_chttp2_hptbl* tbl, uint32_t new_cap) { - grpc_mdelem* ents = - static_cast(gpr_malloc(sizeof(*ents) * new_cap)); - uint32_t i; - - for (i = 0; i < tbl->num_ents; i++) { - ents[i] = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; +void HPackTable::Rebuild(uint32_t new_cap) { + EntriesVec entries; + entries.resize(new_cap); + for (size_t i = 0; i < num_entries_; i++) { + entries[i] = entries_[(first_entry_ + i) % entries_.size()]; } - gpr_free(tbl->ents); - tbl->ents = ents; - tbl->cap_entries = new_cap; - tbl->first_ent = 0; + first_entry_ = 0; + entries_.swap(entries); } -void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl* tbl, - uint32_t max_bytes) { - if (tbl->max_bytes == max_bytes) { +void HPackTable::SetMaxBytes(uint32_t max_bytes) { + if (max_bytes_ == max_bytes) { return; } if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { gpr_log(GPR_INFO, "Update hpack parser max size to %d", max_bytes); } - while (tbl->mem_used > max_bytes) { - evict1(tbl); + while (mem_used_ > max_bytes) { + EvictOne(); } - tbl->max_bytes = max_bytes; + max_bytes_ = max_bytes; } -grpc_error_handle grpc_chttp2_hptbl_set_current_table_size( - grpc_chttp2_hptbl* tbl, uint32_t bytes) { - if (tbl->current_table_bytes == bytes) { +grpc_error_handle HPackTable::SetCurrentTableSize(uint32_t bytes) { + if (current_table_bytes_ == bytes) { return GRPC_ERROR_NONE; } - if (bytes > tbl->max_bytes) { + if (bytes > max_bytes_) { return GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrFormat( "Attempt to make hpack table %d bytes when max is %d bytes", bytes, - tbl->max_bytes) + max_bytes_) .c_str()); } if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { gpr_log(GPR_INFO, "Update hpack parser table size to %d", bytes); } - while (tbl->mem_used > bytes) { - evict1(tbl); - } - tbl->current_table_bytes = bytes; - tbl->max_entries = grpc_chttp2_hptbl::entries_for_bytes(bytes); - if (tbl->max_entries > tbl->cap_entries) { - rebuild_ents(tbl, GPR_MAX(tbl->max_entries, 2 * tbl->cap_entries)); - } else if (tbl->max_entries < tbl->cap_entries / 3) { - uint32_t new_cap = GPR_MAX(tbl->max_entries, 16u); - if (new_cap != tbl->cap_entries) { - rebuild_ents(tbl, new_cap); + while (mem_used_ > bytes) { + EvictOne(); + } + current_table_bytes_ = bytes; + max_entries_ = EntriesForBytes(bytes); + if (max_entries_ > entries_.size()) { + Rebuild(max_entries_); + } else if (max_entries_ < entries_.size() / 3) { + // TODO(ctiller): move to resource quota system, only shrink under memory + // pressure + uint32_t new_cap = std::max(max_entries_, kInlineEntries); + if (new_cap != entries_.size()) { + Rebuild(new_cap); } } return GRPC_ERROR_NONE; } -grpc_error_handle grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl, - grpc_mdelem md) { +grpc_error_handle HPackTable::Add(grpc_mdelem md) { /* determine how many bytes of buffer this entry represents */ size_t elem_bytes = GRPC_SLICE_LENGTH(GRPC_MDKEY(md)) + - GRPC_SLICE_LENGTH(GRPC_MDVALUE(md)) + - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; + GRPC_SLICE_LENGTH(GRPC_MDVALUE(md)) + kEntryOverhead; - if (tbl->current_table_bytes > tbl->max_bytes) { + if (current_table_bytes_ > max_bytes_) { return GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrFormat( "HPACK max table size reduced to %d but not reflected by hpack " "stream (still at %d)", - tbl->max_bytes, tbl->current_table_bytes) + max_bytes_, current_table_bytes_) .c_str()); } - /* we can't add elements bigger than the max table size */ - if (elem_bytes > tbl->current_table_bytes) { - /* HPACK draft 10 section 4.4 states: - * If the size of the new entry is less than or equal to the maximum - * size, that entry is added to the table. It is not an error to - * attempt to add an entry that is larger than the maximum size; an - * attempt to add an entry larger than the entire table causes - * the table - * to be emptied of all existing entries, and results in an - * empty table. - */ - while (tbl->num_ents) { - evict1(tbl); + // we can't add elements bigger than the max table size + if (elem_bytes > current_table_bytes_) { + // HPACK draft 10 section 4.4 states: + // If the size of the new entry is less than or equal to the maximum + // size, that entry is added to the table. It is not an error to + // attempt to add an entry that is larger than the maximum size; an + // attempt to add an entry larger than the entire table causes + // the table to be emptied of all existing entries, and results in an + // empty table. + while (num_entries_) { + EvictOne(); } return GRPC_ERROR_NONE; } - /* evict entries to ensure no overflow */ - while (elem_bytes > - static_cast(tbl->current_table_bytes) - tbl->mem_used) { - evict1(tbl); + // evict entries to ensure no overflow + while (elem_bytes > static_cast(current_table_bytes_) - mem_used_) { + EvictOne(); } - /* copy the finalized entry in */ - tbl->ents[(tbl->first_ent + tbl->num_ents) % tbl->cap_entries] = + // copy the finalized entry in + entries_[(first_entry_ + num_entries_) % entries_.size()] = GRPC_MDELEM_REF(md); - /* update accounting values */ - tbl->num_ents++; - tbl->mem_used += static_cast(elem_bytes); + // update accounting values + num_entries_++; + mem_used_ += static_cast(elem_bytes); return GRPC_ERROR_NONE; } -grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( - const grpc_chttp2_hptbl* tbl, grpc_mdelem md) { - grpc_chttp2_hptbl_find_result r = {0, 0}; - uint32_t i; - - /* See if the string is in the static table */ - for (i = 0; i < GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { - grpc_mdelem ent = grpc_static_mdelem_manifested()[i]; - if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; - r.index = i + 1u; - r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); - if (r.has_value) return r; - } - - /* Scan the dynamic table */ - for (i = 0; i < tbl->num_ents; i++) { - uint32_t idx = static_cast(tbl->num_ents - i + - GRPC_CHTTP2_LAST_STATIC_ENTRY); - grpc_mdelem ent = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; - if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; - r.index = idx; - r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); - if (r.has_value) return r; - } - - return r; -} +} // namespace grpc_core static size_t get_base64_encoded_size(size_t raw_length) { static const uint8_t tail_xtra[3] = {0, 2, 3}; diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h index 14a77932a2e..25129a9b11b 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.h +++ b/src/core/ext/transport/chttp2/transport/hpack_table.h @@ -27,98 +27,111 @@ #include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/static_metadata.h" -/* HPACK header table */ - -/* last index in the static table */ -#define GRPC_CHTTP2_LAST_STATIC_ENTRY 61 - -/* Initial table size as per the spec */ -#define GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE 4096 -/* Maximum table size that we'll use */ -#define GRPC_CHTTP2_MAX_HPACK_TABLE_SIZE GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE -/* Per entry overhead bytes as per the spec */ -#define GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD 32 -#if 0 -/* Maximum number of entries we could possibly fit in the table, given defined - overheads */ -#define GRPC_CHTTP2_MAX_TABLE_COUNT \ - ((GRPC_CHTTP2_MAX_HPACK_TABLE_SIZE + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / \ - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD) -#endif - -/* hpack decoder table */ -struct grpc_chttp2_hptbl { - static uint32_t entries_for_bytes(uint32_t bytes) { - return (bytes + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; - } - static constexpr uint32_t kInitialCapacity = - (GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - - 1) / - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; - - grpc_chttp2_hptbl() { - GPR_DEBUG_ASSERT(!ents); - constexpr uint32_t AllocSize = sizeof(*ents) * kInitialCapacity; - ents = static_cast(gpr_malloc(AllocSize)); - memset(ents, 0, AllocSize); - } +namespace grpc_core { - /* the first used entry in ents */ - uint32_t first_ent = 0; - /* how many entries are in the table */ - uint32_t num_ents = 0; - /* the amount of memory used by the table, according to the hpack algorithm */ - uint32_t mem_used = 0; - /* the max memory allowed to be used by the table, according to the hpack - algorithm */ - uint32_t max_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; - /* the currently agreed size of the table, according to the hpack algorithm */ - uint32_t current_table_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; - /* Maximum number of entries we could possibly fit in the table, given defined - overheads */ - uint32_t max_entries = kInitialCapacity; - /* Number of entries allocated in ents */ - uint32_t cap_entries = kInitialCapacity; - /* a circular buffer of headers - this is stored in the opposite order to - what hpack specifies, in order to simplify table management a little... - meaning lookups need to SUBTRACT from the end position */ - grpc_mdelem* ents = nullptr; -}; +namespace hpack_table_detail { +// Per entry overhead bytes as per the spec +static constexpr uint32_t kEntryOverhead = 32; +// Initial table size as per the spec +static constexpr uint32_t kInitialTableSize = 4096; + +static constexpr uint32_t EntriesForBytes(uint32_t bytes) noexcept { + return (bytes + kEntryOverhead - 1) / kEntryOverhead; +} + +static constexpr uint32_t kInlineEntries = EntriesForBytes(kInitialTableSize); +} // namespace hpack_table_detail + +// HPACK header table +class HPackTable { + public: + // last index in the static table + static constexpr uint32_t kLastStaticEntry = 61; + // Initial table size as per the spec + static constexpr uint32_t kInitialTableSize = + hpack_table_detail::kInitialTableSize; + // Per entry overhead bytes as per the spec + static constexpr uint32_t kEntryOverhead = hpack_table_detail::kEntryOverhead; + + HPackTable(); + ~HPackTable(); + + HPackTable(const HPackTable&); + HPackTable& operator=(const HPackTable&); + + void SetMaxBytes(uint32_t max_bytes); + grpc_error_handle SetCurrentTableSize(uint32_t bytes); + + // Lookup, but don't ref. + grpc_mdelem Peek(uint32_t index) const { return Lookup(index); } + // Lookup, taking a ref if found. + grpc_mdelem Fetch(uint32_t index) const { return Lookup(index); } + + // add a table entry to the index + grpc_error_handle Add(grpc_mdelem md) GRPC_MUST_USE_RESULT; -void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl* tbl); -void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl* tbl, - uint32_t max_bytes); -grpc_error_handle grpc_chttp2_hptbl_set_current_table_size( - grpc_chttp2_hptbl* tbl, uint32_t bytes); - -/* lookup a table entry based on its hpack index */ -grpc_mdelem grpc_chttp2_hptbl_lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index); -grpc_mdelem grpc_chttp2_hptbl_lookup_ref_dynamic_index( - const grpc_chttp2_hptbl* tbl, uint32_t tbl_index); -template -inline grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl, - uint32_t index) { - /* Static table comes first, just return an entry from it. - NB: This imposes the constraint that the first - GRPC_CHTTP2_LAST_STATIC_ENTRY entries in the core static metadata table - must follow the hpack standard. If that changes, we *must* not rely on - reading the core static metadata table here; at that point we'd need our - own singleton static metadata in the correct order. */ - if (index <= GRPC_CHTTP2_LAST_STATIC_ENTRY) { - return grpc_static_mdelem_manifested()[index - 1]; - } else { - if (take_ref) { - return grpc_chttp2_hptbl_lookup_ref_dynamic_index(tbl, index); + // Current entry count in the table. + uint32_t num_entries() const { return num_entries_; } + + private: + using EntriesVec = + absl::InlinedVector; + + /* lookup a table entry based on its hpack index */ + template + grpc_mdelem Lookup(uint32_t index) const { + // Static table comes first, just return an entry from it. + // NB: This imposes the constraint that the first + // GRPC_CHTTP2_LAST_STATIC_ENTRY entries in the core static metadata table + // must follow the hpack standard. If that changes, we *must* not rely on + // reading the core static metadata table here; at that point we'd need our + // own singleton static metadata in the correct order. + if (index <= kLastStaticEntry) { + return grpc_static_mdelem_manifested()[index - 1]; } else { - return grpc_chttp2_hptbl_lookup_dynamic_index(tbl, index); + return LookupDynamic(index); } } -} -/* add a table entry to the index */ -grpc_error_handle grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl, - grpc_mdelem md) GRPC_MUST_USE_RESULT; + + template + grpc_mdelem LookupDynamic(uint32_t index) const { + // Not static - find the value in the list of valid entries + const uint32_t tbl_index = index - (kLastStaticEntry + 1); + if (tbl_index < num_entries_) { + uint32_t offset = + (num_entries_ - 1u - tbl_index + first_entry_) % entries_.size(); + grpc_mdelem md = entries_[offset]; + if (take_ref) { + GRPC_MDELEM_REF(md); + } + return md; + } + // Invalid entry: return error + return GRPC_MDNULL; + } + + void EvictOne(); + void Rebuild(uint32_t new_cap); + + // The first used entry in ents. + uint32_t first_entry_ = 0; + // How many entries are in the table. + uint32_t num_entries_ = 0; + // The amount of memory used by the table, according to the hpack algorithm + uint32_t mem_used_ = 0; + // The max memory allowed to be used by the table, according to the hpack + // algorithm. + uint32_t max_bytes_ = kInitialTableSize; + // The currently agreed size of the table, according to the hpack algorithm. + uint32_t current_table_bytes_ = kInitialTableSize; + // Maximum number of entries we could possibly fit in the table, given defined + // overheads. + uint32_t max_entries_ = hpack_table_detail::kInlineEntries; + // HPack table entries + EntriesVec entries_; +}; + +} // namespace grpc_core size_t grpc_chttp2_get_size_in_hpack_table(grpc_mdelem elem, bool use_true_binary_metadata); @@ -130,19 +143,10 @@ inline uintptr_t grpc_chttp2_get_static_hpack_table_index(grpc_mdelem md) { uintptr_t index = reinterpret_cast(GRPC_MDELEM_DATA(md)) - grpc_static_mdelem_table(); - if (index < GRPC_CHTTP2_LAST_STATIC_ENTRY) { + if (index < grpc_core::HPackTable::kLastStaticEntry) { return index + 1; // Hpack static metadata element indices start at 1 } return 0; } -/* Find a key/value pair in the table... returns the index in the table of the - most similar entry, or 0 if the value was not found */ -struct grpc_chttp2_hptbl_find_result { - uint32_t index; - int has_value; -}; -grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( - const grpc_chttp2_hptbl* tbl, grpc_mdelem md); - #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index 5d913358105..2b9fb6bba95 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -767,8 +767,7 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) { if (t->incoming_frame_flags & GRPC_CHTTP2_FLAG_ACK) { memcpy(t->settings[GRPC_ACKED_SETTINGS], t->settings[GRPC_SENT_SETTINGS], GRPC_CHTTP2_NUM_SETTINGS * sizeof(uint32_t)); - grpc_chttp2_hptbl_set_max_bytes( - t->hpack_parser.hpack_table(), + t->hpack_parser.hpack_table()->SetMaxBytes( t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); t->sent_local_settings = false; diff --git a/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc b/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc index 7be46ab6cb4..3914fc7d92b 100644 --- a/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc +++ b/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc @@ -45,7 +45,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { grpc_core::HPackParser parser; parser.BeginFrame(onhdr, grpc_core::HPackParser::Boundary::None, grpc_core::HPackParser::Priority::None); - GRPC_ERROR_UNREF(parser.Parse(grpc_slice_from_static_buffer(data, size))); + GRPC_ERROR_UNREF( + parser.Parse(grpc_slice_from_static_buffer(data, size), true)); } grpc_shutdown(); return 0; diff --git a/test/core/transport/chttp2/hpack_parser_test.cc b/test/core/transport/chttp2/hpack_parser_test.cc index 1328366f1b8..cdc8f6bca18 100644 --- a/test/core/transport/chttp2/hpack_parser_test.cc +++ b/test/core/transport/chttp2/hpack_parser_test.cc @@ -65,7 +65,7 @@ static void test_vector(grpc_core::HPackParser* parser, for (i = 0; i < nslices; i++) { grpc_core::ExecCtx exec_ctx; - GPR_ASSERT(parser->Parse(slices[i]) == GRPC_ERROR_NONE); + GPR_ASSERT(parser->Parse(slices[i], i == nslices - 1) == GRPC_ERROR_NONE); } for (i = 0; i < nslices; i++) { @@ -158,8 +158,8 @@ static void test_vectors(grpc_slice_split_mode mode) { { grpc_core::HPackParser parser; - grpc_chttp2_hptbl_set_max_bytes(parser.hpack_table(), 256); - grpc_chttp2_hptbl_set_current_table_size(parser.hpack_table(), 256); + parser.hpack_table()->SetMaxBytes(256); + parser.hpack_table()->SetCurrentTableSize(256); /* D.5.1 */ test_vector(&parser, mode, "4803 3330 3258 0770 7269 7661 7465 611d" @@ -199,8 +199,8 @@ static void test_vectors(grpc_slice_split_mode mode) { { grpc_core::HPackParser parser; - grpc_chttp2_hptbl_set_max_bytes(parser.hpack_table(), 256); - grpc_chttp2_hptbl_set_current_table_size(parser.hpack_table(), 256); + parser.hpack_table()->SetMaxBytes(256); + parser.hpack_table()->SetCurrentTableSize(256); /* D.6.1 */ test_vector(&parser, mode, "4882 6402 5885 aec3 771a 4b61 96d0 7abe" diff --git a/test/core/transport/chttp2/hpack_table_test.cc b/test/core/transport/chttp2/hpack_table_test.cc index c8410c61866..04b69a9b633 100644 --- a/test/core/transport/chttp2/hpack_table_test.cc +++ b/test/core/transport/chttp2/hpack_table_test.cc @@ -36,21 +36,23 @@ #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x) -static void assert_str(const grpc_chttp2_hptbl* /*tbl*/, grpc_slice mdstr, +using grpc_core::HPackTable; + +static void assert_str(const HPackTable* /*tbl*/, grpc_slice mdstr, const char* str) { GPR_ASSERT(grpc_slice_str_cmp(mdstr, str) == 0); } -static void assert_index(const grpc_chttp2_hptbl* tbl, uint32_t idx, - const char* key, const char* value) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(tbl, idx); +static void assert_index(const HPackTable* tbl, uint32_t idx, const char* key, + const char* value) { + grpc_mdelem md = tbl->Peek(idx); assert_str(tbl, GRPC_MDKEY(md), key); assert_str(tbl, GRPC_MDVALUE(md), value); } static void test_static_lookup(void) { grpc_core::ExecCtx exec_ctx; - grpc_chttp2_hptbl tbl; + HPackTable tbl; LOG_TEST("test_static_lookup"); assert_index(&tbl, 1, ":authority", ""); @@ -114,12 +116,10 @@ static void test_static_lookup(void) { assert_index(&tbl, 59, "vary", ""); assert_index(&tbl, 60, "via", ""); assert_index(&tbl, 61, "www-authenticate", ""); - - grpc_chttp2_hptbl_destroy(&tbl); } static void test_many_additions(void) { - grpc_chttp2_hptbl tbl; + HPackTable tbl; int i; LOG_TEST("test_many_additions"); @@ -132,135 +132,17 @@ static void test_many_additions(void) { std::string value = absl::StrCat("VALUE:", i); elem = grpc_mdelem_from_slices(grpc_slice_from_cpp_string(key), grpc_slice_from_cpp_string(value)); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); + GPR_ASSERT(tbl.Add(elem) == GRPC_ERROR_NONE); GRPC_MDELEM_UNREF(elem); - assert_index(&tbl, 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY, key.c_str(), + assert_index(&tbl, 1 + HPackTable::kLastStaticEntry, key.c_str(), value.c_str()); if (i) { std::string key = absl::StrCat("K:", i - 1); std::string value = absl::StrCat("VALUE:", i - 1); - assert_index(&tbl, 2 + GRPC_CHTTP2_LAST_STATIC_ENTRY, key.c_str(), + assert_index(&tbl, 2 + HPackTable::kLastStaticEntry, key.c_str(), value.c_str()); } } - - grpc_chttp2_hptbl_destroy(&tbl); -} - -static grpc_chttp2_hptbl_find_result find_simple(grpc_chttp2_hptbl* tbl, - const char* key, - const char* value) { - grpc_core::ExecCtx exec_ctx; - grpc_mdelem md = grpc_mdelem_from_slices( - grpc_slice_from_copied_string(key), grpc_slice_from_copied_string(value)); - grpc_chttp2_hptbl_find_result r = grpc_chttp2_hptbl_find(tbl, md); - GRPC_MDELEM_UNREF(md); - - return r; -} - -static void test_find(void) { - grpc_core::ExecCtx exec_ctx; - grpc_chttp2_hptbl tbl; - uint32_t i; - char buffer[32]; - grpc_mdelem elem; - grpc_chttp2_hptbl_find_result r; - - LOG_TEST("test_find"); - - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("abc"), - grpc_slice_from_static_string("xyz")); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("abc"), - grpc_slice_from_static_string("123")); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("x"), - grpc_slice_from_static_string("1")); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - - r = find_simple(&tbl, "abc", "123"); - GPR_ASSERT(r.index == 2 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "abc", "xyz"); - GPR_ASSERT(r.index == 3 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "x", "1"); - GPR_ASSERT(r.index == 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "x", "2"); - GPR_ASSERT(r.index == 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "vary", "some-vary-arg"); - GPR_ASSERT(r.index == 59); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "accept-encoding", "gzip, deflate"); - GPR_ASSERT(r.index == 16); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "accept-encoding", "gzip"); - GPR_ASSERT(r.index == 16); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, ":method", "GET"); - GPR_ASSERT(r.index == 2); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, ":method", "POST"); - GPR_ASSERT(r.index == 3); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, ":method", "PUT"); - GPR_ASSERT(r.index == 2 || r.index == 3); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "this-does-not-exist", ""); - GPR_ASSERT(r.index == 0); - GPR_ASSERT(r.has_value == 0); - - /* overflow the string buffer, check find still works */ - for (i = 0; i < 10000; i++) { - int64_ttoa(i, buffer); - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("test"), - grpc_slice_from_copied_string(buffer)); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - } - - r = find_simple(&tbl, "abc", "123"); - GPR_ASSERT(r.index == 0); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "test", "9999"); - GPR_ASSERT(r.index == 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "test", "9998"); - GPR_ASSERT(r.index == 2 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - for (i = 0; i < tbl.num_ents; i++) { - uint32_t expect = 9999 - i; - int64_ttoa(expect, buffer); - - r = find_simple(&tbl, "test", buffer); - GPR_ASSERT(r.index == i + 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - } - - r = find_simple(&tbl, "test", "10000"); - GPR_ASSERT(r.index != 0); - GPR_ASSERT(r.has_value == 0); - - grpc_chttp2_hptbl_destroy(&tbl); } int main(int argc, char** argv) { @@ -268,7 +150,6 @@ int main(int argc, char** argv) { grpc_init(); test_static_lookup(); test_many_additions(); - test_find(); grpc_shutdown(); return 0; } diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 5a29df607da..a7e645666ae 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -461,11 +461,11 @@ static void BM_HpackParserParseHeader(benchmark::State& state) { grpc_core::HPackParser::Boundary::None, grpc_core::HPackParser::Priority::None); for (auto slice : init_slices) { - GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice)); + GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice, false)); } while (state.KeepRunning()) { for (auto slice : benchmark_slices) { - GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice)); + GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice, false)); } grpc_core::ExecCtx::Get()->Flush(); // Recreate arena every 4k iterations to avoid oom diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 129a65e683b..61c359e3a7d 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1780,10 +1780,12 @@ src/core/lib/gprpp/global_config_generic.h \ src/core/lib/gprpp/host_port.cc \ src/core/lib/gprpp/host_port.h \ src/core/lib/gprpp/manual_constructor.h \ +src/core/lib/gprpp/match.h \ src/core/lib/gprpp/memory.h \ src/core/lib/gprpp/mpscq.cc \ src/core/lib/gprpp/mpscq.h \ src/core/lib/gprpp/orphanable.h \ +src/core/lib/gprpp/overload.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ src/core/lib/gprpp/stat.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 9a298adef3b..7a46916a47f 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1619,10 +1619,12 @@ src/core/lib/gprpp/global_config_generic.h \ src/core/lib/gprpp/host_port.cc \ src/core/lib/gprpp/host_port.h \ src/core/lib/gprpp/manual_constructor.h \ +src/core/lib/gprpp/match.h \ src/core/lib/gprpp/memory.h \ src/core/lib/gprpp/mpscq.cc \ src/core/lib/gprpp/mpscq.h \ src/core/lib/gprpp/orphanable.h \ +src/core/lib/gprpp/overload.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ src/core/lib/gprpp/stat.h \