From bbea27de4631fee3b8d32c9e5bde1369dc7b3788 Mon Sep 17 00:00:00 2001 From: Craig Tiller Date: Fri, 13 Aug 2021 10:57:42 -0700 Subject: [PATCH] Reland HPACK parsing changes (#26997) * Revert "Revert "HPACK Table --> C++ (#26851)" (#26995)" This reverts commit 840bcce9c4e9a6eaf2c6bd3ce65a0764d309bc31. * fix bad parsing of trailing === in binary metadata --- BUILD | 1 + CMakeLists.txt | 10 +- build_autogenerated.yaml | 6 + gRPC-C++.podspec | 5 + gRPC-Core.podspec | 5 + grpc.gemspec | 2 + grpc.gyp | 2 + package.xml | 2 + .../chttp2/transport/hpack_encoder.cc | 7 +- .../chttp2/transport/hpack_parser.cc | 1813 +++++++---------- .../transport/chttp2/transport/hpack_parser.h | 215 +- .../transport/chttp2/transport/hpack_table.cc | 201 +- .../transport/chttp2/transport/hpack_table.h | 198 +- .../ext/transport/chttp2/transport/parsing.cc | 3 +- .../transport/chttp2/binary-metadata.headers | 2 + .../chttp2/hpack_parser_fuzzer_test.cc | 3 +- .../transport/chttp2/hpack_parser_test.cc | 26 +- .../core/transport/chttp2/hpack_table_test.cc | 141 +- test/cpp/microbenchmarks/bm_chttp2_hpack.cc | 4 +- tools/doxygen/Doxyfile.c++.internal | 2 + tools/doxygen/Doxyfile.core.internal | 2 + 21 files changed, 1072 insertions(+), 1578 deletions(-) create mode 100644 test/core/transport/chttp2/binary-metadata.headers diff --git a/BUILD b/BUILD index fbb11ff1745..9beffae464e 100644 --- a/BUILD +++ b/BUILD @@ -2647,6 +2647,7 @@ grpc_cc_library( "grpc_http_filters", "grpc_trace", "grpc_transport_chttp2_alpn", + "match", "popularity_count", ], ) diff --git a/CMakeLists.txt b/CMakeLists.txt index 276b25b8a22..feec91f63f3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2132,6 +2132,7 @@ target_link_libraries(grpc absl::inlined_vector absl::bind_front absl::statusor + absl::variant gpr ${_gRPC_SSL_LIBRARIES} address_sorting @@ -2688,6 +2689,7 @@ target_link_libraries(grpc_unsecure absl::inlined_vector absl::bind_front absl::statusor + absl::variant gpr address_sorting ) @@ -16283,7 +16285,7 @@ generate_pkgconfig( "gRPC" "high performance general RPC framework" "${gRPC_CORE_VERSION}" - "gpr openssl absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "gpr openssl absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc -laddress_sorting -lre2 -lupb -lcares -lz" "" "grpc.pc") @@ -16293,7 +16295,7 @@ generate_pkgconfig( "gRPC unsecure" "high performance general RPC framework without SSL" "${gRPC_CORE_VERSION}" - "gpr absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "gpr absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc_unsecure" "" "grpc_unsecure.pc") @@ -16303,7 +16305,7 @@ generate_pkgconfig( "gRPC++" "C++ wrapper for gRPC" "${gRPC_CPP_VERSION}" - "grpc absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "grpc absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc++" "" "grpc++.pc") @@ -16313,7 +16315,7 @@ generate_pkgconfig( "gRPC++ unsecure" "C++ wrapper for gRPC without SSL" "${gRPC_CPP_VERSION}" - "grpc_unsecure absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time" + "grpc_unsecure absl_base absl_bind_front absl_cord absl_core_headers absl_flat_hash_map absl_inlined_vector absl_memory absl_optional absl_status absl_statusor absl_str_format absl_strings absl_synchronization absl_time absl_variant" "-lgrpc++_unsecure" "" "grpc++_unsecure.pc") diff --git a/build_autogenerated.yaml b/build_autogenerated.yaml index 42e7dc7e64f..261de378dcc 100644 --- a/build_autogenerated.yaml +++ b/build_autogenerated.yaml @@ -739,7 +739,9 @@ libs: - src/core/lib/event_engine/sockaddr.h - src/core/lib/gprpp/atomic.h - src/core/lib/gprpp/dual_ref_counted.h + - src/core/lib/gprpp/match.h - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/overload.h - src/core/lib/gprpp/ref_counted.h - src/core/lib/gprpp/ref_counted_ptr.h - src/core/lib/http/format_request.h @@ -1510,6 +1512,7 @@ libs: - absl/container:inlined_vector - absl/functional:bind_front - absl/status:statusor + - absl/types:variant - gpr - libssl - address_sorting @@ -1776,7 +1779,9 @@ libs: - src/core/lib/event_engine/sockaddr.h - src/core/lib/gprpp/atomic.h - src/core/lib/gprpp/dual_ref_counted.h + - src/core/lib/gprpp/match.h - src/core/lib/gprpp/orphanable.h + - src/core/lib/gprpp/overload.h - src/core/lib/gprpp/ref_counted.h - src/core/lib/gprpp/ref_counted_ptr.h - src/core/lib/http/format_request.h @@ -2192,6 +2197,7 @@ libs: - absl/container:inlined_vector - absl/functional:bind_front - absl/status:statusor + - absl/types:variant - gpr - address_sorting baselib: true diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 91d4ff16e57..8c9ba5b380d 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -202,6 +202,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/synchronization/synchronization', abseil_version ss.dependency 'abseil/time/time', abseil_version ss.dependency 'abseil/types/optional', abseil_version + ss.dependency 'abseil/types/variant', abseil_version ss.source_files = 'src/core/ext/filters/client_channel/backend_metric.h', 'src/core/ext/filters/client_channel/backup_poller.h', @@ -553,9 +554,11 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/global_config_generic.h', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', @@ -1213,9 +1216,11 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/global_config_generic.h', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index e788b50d7b8..9a13d013c97 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -192,6 +192,7 @@ Pod::Spec.new do |s| ss.dependency 'abseil/synchronization/synchronization', abseil_version ss.dependency 'abseil/time/time', abseil_version ss.dependency 'abseil/types/optional', abseil_version + ss.dependency 'abseil/types/variant', abseil_version ss.compiler_flags = '-DBORINGSSL_PREFIX=GRPC -Wno-unreachable-code -Wno-shorten-64-to-32' ss.source_files = 'src/core/ext/filters/census/grpc_context.cc', @@ -928,10 +929,12 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/host_port.cc', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.cc', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', @@ -1800,9 +1803,11 @@ Pod::Spec.new do |s| 'src/core/lib/gprpp/global_config_generic.h', 'src/core/lib/gprpp/host_port.h', 'src/core/lib/gprpp/manual_constructor.h', + 'src/core/lib/gprpp/match.h', 'src/core/lib/gprpp/memory.h', 'src/core/lib/gprpp/mpscq.h', 'src/core/lib/gprpp/orphanable.h', + 'src/core/lib/gprpp/overload.h', 'src/core/lib/gprpp/ref_counted.h', 'src/core/lib/gprpp/ref_counted_ptr.h', 'src/core/lib/gprpp/stat.h', diff --git a/grpc.gemspec b/grpc.gemspec index 7130af7b026..968acec1718 100644 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -842,10 +842,12 @@ Gem::Specification.new do |s| s.files += %w( src/core/lib/gprpp/host_port.cc ) s.files += %w( src/core/lib/gprpp/host_port.h ) s.files += %w( src/core/lib/gprpp/manual_constructor.h ) + s.files += %w( src/core/lib/gprpp/match.h ) s.files += %w( src/core/lib/gprpp/memory.h ) s.files += %w( src/core/lib/gprpp/mpscq.cc ) s.files += %w( src/core/lib/gprpp/mpscq.h ) s.files += %w( src/core/lib/gprpp/orphanable.h ) + s.files += %w( src/core/lib/gprpp/overload.h ) s.files += %w( src/core/lib/gprpp/ref_counted.h ) s.files += %w( src/core/lib/gprpp/ref_counted_ptr.h ) s.files += %w( src/core/lib/gprpp/stat.h ) diff --git a/grpc.gyp b/grpc.gyp index f359f773e60..53cca3613ac 100644 --- a/grpc.gyp +++ b/grpc.gyp @@ -473,6 +473,7 @@ 'absl/container:inlined_vector', 'absl/functional:bind_front', 'absl/status:statusor', + 'absl/types:variant', 'gpr', 'address_sorting', ], @@ -1132,6 +1133,7 @@ 'absl/container:inlined_vector', 'absl/functional:bind_front', 'absl/status:statusor', + 'absl/types:variant', 'gpr', 'address_sorting', ], diff --git a/package.xml b/package.xml index 95c66e24b1a..ed1941e905b 100644 --- a/package.xml +++ b/package.xml @@ -822,10 +822,12 @@ + + diff --git a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc index 932415436d7..04f289dfbb2 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_encoder.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_encoder.cc @@ -586,7 +586,7 @@ static void GPR_ATTRIBUTE_NOINLINE hpack_enc_log(grpc_mdelem elem) { } static uint32_t dynidx(grpc_chttp2_hpack_compressor* c, uint32_t elem_index) { - return 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY + c->tail_remote_index + + return 1 + grpc_core::HPackTable::kLastStaticEntry + c->tail_remote_index + c->table_elems - elem_index; } @@ -833,7 +833,7 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c, if (is_static && (static_index = reinterpret_cast(GRPC_MDELEM_DATA(md)) - ->StaticIndex()) < GRPC_CHTTP2_LAST_STATIC_ENTRY) { + ->StaticIndex()) < grpc_core::HPackTable::kLastStaticEntry) { emit_indexed(c, static_cast(static_index + 1), &st); } else { hpack_enc(c, md, &st); @@ -847,7 +847,8 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c, if (is_static && (static_index = reinterpret_cast( GRPC_MDELEM_DATA(l->md)) - ->StaticIndex()) < GRPC_CHTTP2_LAST_STATIC_ENTRY) { + ->StaticIndex()) < + grpc_core::HPackTable::kLastStaticEntry) { emit_indexed(c, static_cast(static_index + 1), &st); } else { hpack_enc(c, l->md, &st); diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.cc b/src/core/ext/transport/chttp2/transport/hpack_parser.cc index 3d6e1aa346b..d727b6cf2dd 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.cc @@ -34,6 +34,7 @@ #include "src/core/ext/transport/chttp2/transport/bin_encoder.h" #include "src/core/lib/debug/stats.h" #include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/match.h" #include "src/core/lib/profiling/timers.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -52,20 +53,6 @@ namespace grpc_core { DebugOnlyTraceFlag grpc_trace_chttp2_hpack_parser(false, "chttp2_hpack_parser"); -/* How parsing works: - - The parser object keeps track of a function pointer which represents the - current parse state. - - Each time new bytes are presented, we call into the current state, which - recursively parses until all bytes in the given chunk are exhausted. - - The parse state that terminates then saves its function pointer to be the - current state so that it can resume when more bytes are available. - - It's expected that most optimizing compilers will turn this code into - a set of indirect jumps, and so not waste stack space. */ - /* state table for huffman decoding: given a state, gives an index/16 into next_sub_tbl. Taking that index and adding the value of the nibble being considered returns the next state. @@ -464,1121 +451,879 @@ struct Base64InverseTable { static GRPC_HPACK_CONSTEXPR_VALUE Base64InverseTable kBase64InverseTable; } // namespace -void HPackParser::FinishFrame() { - sink_ = Sink(); - dynamic_table_updates_allowed_ = 2; -} - -void GPR_ATTRIBUTE_NOINLINE HPackParser::LogHeader(grpc_mdelem md) { - char* k = grpc_slice_to_c_string(GRPC_MDKEY(md)); - char* v = nullptr; - if (grpc_is_binary_header_internal(GRPC_MDKEY(md))) { - v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX); - } else { - v = grpc_slice_to_c_string(GRPC_MDVALUE(md)); +// Input tracks the current byte through the input data and provides it +// via a simple stream interface. +class HPackParser::Input { + public: + Input(grpc_slice_refcount* current_slice_refcount, const uint8_t* begin, + const uint8_t* end) + : current_slice_refcount_(current_slice_refcount), + begin_(begin), + end_(end), + frontier_(begin) {} + + // If input is backed by a slice, retrieve its refcount. If not, return + // nullptr. + grpc_slice_refcount* slice_refcount() { return current_slice_refcount_; } + + // Have we reached the end of input? + bool end_of_stream() const { return begin_ == end_; } + // How many bytes until end of input + size_t remaining() const { return end_ - begin_; } + // Current position, as a pointer + const uint8_t* cur_ptr() const { return begin_; } + // End position, as a pointer + const uint8_t* end_ptr() const { return end_; } + // Move read position forward by n, unchecked + void Advance(size_t n) { begin_ += n; } + + // Retrieve the current character, or nullopt if end of stream + // Do not advance + absl::optional peek() const { + if (end_of_stream()) { + return {}; + } + return *begin_; } - gpr_log( - GPR_INFO, - "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d", - k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md), - grpc_slice_is_interned(GRPC_MDKEY(md)), - grpc_slice_is_interned(GRPC_MDVALUE(md))); - gpr_free(k); - gpr_free(v); -} -/* emission helpers */ -template -grpc_error_handle HPackParser::FinishHeader(grpc_mdelem md) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { - LogHeader(md); - } - if (action == TableAction::kAddToTable) { - GPR_DEBUG_ASSERT(GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_INTERNED || - GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC); - grpc_error_handle err = grpc_chttp2_hptbl_add(&table_, md); - if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) return err; + // Retrieve and advance past the current character, or return nullopt if end + // of stream + absl::optional Next() { + if (end_of_stream()) { + return UnexpectedEOF(absl::optional()); + } + return *begin_++; } - return sink_(md); -} -UnmanagedMemorySlice HPackParser::String::TakeExtern() { - UnmanagedMemorySlice s; - if (!copied_) { - GPR_DEBUG_ASSERT(!grpc_slice_is_interned(data_.referenced)); - s = static_cast(data_.referenced); - copied_ = true; - data_.referenced = UnmanagedMemorySlice(); - } else { - s = UnmanagedMemorySlice(data_.copied.str, data_.copied.length); + // Helper to parse a varint delta on top of value, return nullopt on failure + // (setting error) + absl::optional ParseVarint(uint32_t value) { + // TODO(ctiller): break out a variant of this when we know there are at + // least 5 bytes in input_ + auto cur = Next(); + if (!cur) return {}; + value += *cur & 0x7f; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + value += (*cur & 0x7f) << 7; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + value += (*cur & 0x7f) << 14; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + value += (*cur & 0x7f) << 21; + if ((*cur & 0x80) == 0) return value; + + cur = Next(); + if (!cur) return {}; + uint32_t c = (*cur) & 0x7f; + // We might overflow here, so we need to be a little careful about the + // addition + if (c > 0xf) return ParseVarintOutOfRange(value, *cur); + const uint32_t add = c << 28; + if (add > 0xffffffffu - value) { + return ParseVarintOutOfRange(value, *cur); + } + value += add; + if ((*cur & 0x80) == 0) return value; + + // Spec weirdness: we can add an infinite stream of 0x80 at the end of a + // varint and still end up with a correctly encoded varint. + do { + cur = Next(); + if (!cur.has_value()) return {}; + } while (*cur == 0x80); + + // BUT... the last byte needs to be 0x00 or we'll overflow dramatically! + if (*cur == 0) return value; + return ParseVarintOutOfRange(value, *cur); } - data_.copied.length = 0; - return s; -} -ManagedMemorySlice HPackParser::String::TakeIntern() { - ManagedMemorySlice s; - if (!copied_) { - s = ManagedMemorySlice(&data_.referenced); - grpc_slice_unref_internal(data_.referenced); - copied_ = true; - data_.referenced = grpc_empty_slice(); - } else { - s = ManagedMemorySlice(data_.copied.str, data_.copied.length); + // Prefix for a string + struct StringPrefix { + // Number of bytes in input for string + uint32_t length; + // Is it huffman compressed + bool huff; + }; + + // Parse a string prefix + absl::optional ParseStringPrefix() { + auto cur = Next(); + if (!cur.has_value()) return {}; + // Huffman if the top bit is 1 + const bool huff = (*cur & 0x80) != 0; + // String length + uint32_t strlen = (*cur & 0x7f); + if (strlen == 0x7f) { + // all ones ==> varint string length + auto v = ParseVarint(0x7f); + if (!v.has_value()) return {}; + strlen = *v; + } + return StringPrefix{strlen, huff}; } - data_.copied.length = 0; - return s; -} -grpc_error_handle HPackParser::parse_next(const uint8_t* cur, - const uint8_t* end) { - state_ = *next_state_++; - return (this->*state_)(cur, end); -} + // Check if we saw an EOF.. must be verified before looking at TakeError + bool eof_error() const { return eof_error_; } -/* begin parsing a header: all functionality is encoded into lookup tables - above */ -grpc_error_handle HPackParser::parse_begin(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_begin; - return GRPC_ERROR_NONE; + // Extract the parse error, leaving the current error as NONE. + grpc_error_handle TakeError() { + grpc_error_handle out = error_; + error_ = GRPC_ERROR_NONE; + return out; } - switch (*cur >> 4) { - // Literal header not indexed. - // First byte format: 0000xxxx - // Where xxxx: - // 0000 - literal key - // 1111 - indexed key, varint encoded index - // other - indexed key, inline encoded index - case 0: - switch (*cur & 0xf) { - case 0: // literal key - return parse_lithdr_notidx_v(cur, end); - case 0xf: // varint encoded key index - return parse_lithdr_notidx_x(cur, end); - default: // inline encoded key index - return parse_lithdr_notidx(cur, end); - } - // Literal header never indexed. - // First byte format: 0001xxxx - // Where xxxx: - // 0000 - literal key - // 1111 - indexed key, varint encoded index - // other - indexed key, inline encoded index - case 1: - switch (*cur & 0xf) { - case 0: // literal key - return parse_lithdr_nvridx_v(cur, end); - case 0xf: // varint encoded key index - return parse_lithdr_nvridx_x(cur, end); - default: // inline encoded key index - return parse_lithdr_nvridx(cur, end); - } - // Update max table size. - // First byte format: 001xxxxx - // Where xxxxx: - // 11111 - max size is varint encoded - // other - max size is stored inline - case 2: - // inline encoded max table size - return parse_max_tbl_size(cur, end); - case 3: - if (*cur == 0x3f) { - // varint encoded max table size - return parse_max_tbl_size_x(cur, end); - } else { - // inline encoded max table size - return parse_max_tbl_size(cur, end); - } - // Literal header with incremental indexing. - // First byte format: 01xxxxxx - // Where xxxxxx: - // 000000 - literal key - // 111111 - indexed key, varint encoded index - // other - indexed key, inline encoded index - case 4: - if (*cur == 0x40) { - // literal key - return parse_lithdr_incidx_v(cur, end); - } - ABSL_FALLTHROUGH_INTENDED; - case 5: - case 6: - // inline encoded key index - return parse_lithdr_incidx(cur, end); - case 7: - if (*cur == 0x7f) { - // varint encoded key index - return parse_lithdr_incidx_x(cur, end); - } else { - // inline encoded key index - return parse_lithdr_incidx(cur, end); - } - // Indexed Header Field Representation - // First byte format: 1xxxxxxx - // Where xxxxxxx: - // 0000000 - illegal - // 1111111 - varint encoded field index - // other - inline encoded field index - case 8: - if (*cur == 0x80) { - // illegal value. - return parse_illegal_op(cur, end); - } - ABSL_FALLTHROUGH_INTENDED; - case 9: - case 10: - case 11: - case 12: - case 13: - case 14: - // inline encoded field index - return parse_indexed_field(cur, end); - case 15: - if (*cur == 0xff) { - // varint encoded field index - return parse_indexed_field_x(cur, end); - } else { - // inline encoded field index - return parse_indexed_field(cur, end); - } + // Set the current error - allows the rest of the code not to need to pass + // around StatusOr<> which would be prohibitive here. + GPR_ATTRIBUTE_NOINLINE void SetError(grpc_error_handle error) { + if (error_ != GRPC_ERROR_NONE || eof_error_) { + GRPC_ERROR_UNREF(error); + return; + } + error_ = error; + begin_ = end_; } - GPR_UNREACHABLE_CODE(abort()); -} -/* stream dependency and prioritization data: we just skip it */ -grpc_error_handle HPackParser::parse_stream_weight(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_weight; - return GRPC_ERROR_NONE; + // If no error is set, set it to the value produced by error_factory. + // Return return_value unchanged. + template + GPR_ATTRIBUTE_NOINLINE T MaybeSetErrorAndReturn(F error_factory, + T return_value) { + if (error_ != GRPC_ERROR_NONE || eof_error_) return return_value; + error_ = error_factory(); + begin_ = end_; + return return_value; } - return (this->*after_prioritization_)(cur + 1, end); -} - -grpc_error_handle HPackParser::parse_stream_dep3(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep3; - return GRPC_ERROR_NONE; + // Set the error to an unexpected eof, and return result (code golfed as this + // is a common case) + template + T UnexpectedEOF(T return_value) { + if (error_ != GRPC_ERROR_NONE) return return_value; + eof_error_ = true; + return return_value; } - return parse_stream_weight(cur + 1, end); -} - -grpc_error_handle HPackParser::parse_stream_dep2(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep2; - return GRPC_ERROR_NONE; + // Update the frontier - signifies we've successfully parsed another element + void UpdateFrontier() { frontier_ = begin_; } + + // Get the frontier - for buffering should we fail due to eof + const uint8_t* frontier() const { return frontier_; } + + private: + // Helper to set the error to out of range for ParseVarint + absl::optional ParseVarintOutOfRange(uint32_t value, + uint8_t last_byte) { + return MaybeSetErrorAndReturn( + [value, last_byte] { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrFormat( + "integer overflow in hpack integer decoding: have 0x%08x, " + "got byte 0x%02x on byte 5", + value, last_byte) + .c_str()); + }, + absl::optional()); } - return parse_stream_dep3(cur + 1, end); -} + // Refcount if we are backed by a slice + grpc_slice_refcount* current_slice_refcount_; + // Current input point + const uint8_t* begin_; + // End of stream point + const uint8_t* const end_; + // Frontier denotes the first byte past successfully processed input + const uint8_t* frontier_; + // Current error + grpc_error_handle error_ = GRPC_ERROR_NONE; + // If the error was EOF, we flag it here.. + bool eof_error_ = false; +}; -grpc_error_handle HPackParser::parse_stream_dep1(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep1; - return GRPC_ERROR_NONE; +// Helper to parse a string and turn it into a slice with appropriate memory +// management characteristics +class HPackParser::String { + public: + // Helper to specify a string should be internalized + struct Intern {}; + // Helper to specify a string should be externalized + struct Extern {}; + + private: + // Forward declare take functions... we'll need them in the public interface + UnmanagedMemorySlice Take(Extern); + ManagedMemorySlice Take(Intern); + + public: + // If a String is a Slice then unref + ~String() { + if (auto* p = absl::get_if(&value_)) { + grpc_slice_unref_internal(*p); + } } - return parse_stream_dep2(cur + 1, end); -} - -grpc_error_handle HPackParser::parse_stream_dep0(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_stream_dep0; - return GRPC_ERROR_NONE; + // Take the value and leave this empty + // Use Intern/Extern to choose memory management + template + auto Take() -> decltype(this->Take(T())) { + return Take(T()); } - return parse_stream_dep1(cur + 1, end); -} - -grpc_error_handle HPackParser::InvalidHPackIndexError() { - return grpc_error_set_int( - grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Invalid HPACK index received"), - GRPC_ERROR_INT_INDEX, static_cast(index_)), - GRPC_ERROR_INT_SIZE, static_cast(table_.num_ents)); -} - -/* emit an indexed field; jumps to begin the next field on completion */ -grpc_error_handle HPackParser::finish_indexed_field(const uint8_t* cur, - const uint8_t* end) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(&table_, index_); - if (GPR_UNLIKELY(GRPC_MDISNULL(md))) { - return InvalidHPackIndexError(); + String(const String&) = delete; + String& operator=(const String&) = delete; + String(String&& other) noexcept : value_(std::move(other.value_)) { + other.value_ = absl::Span(); } - GRPC_STATS_INC_HPACK_RECV_INDEXED(); - grpc_error_handle err = FinishHeader(md); - if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) return err; - return parse_begin(cur, end); -} - -/* parse an indexed field with index < 127 */ -grpc_error_handle HPackParser::parse_indexed_field(const uint8_t* cur, - const uint8_t* end) { - dynamic_table_updates_allowed_ = 0; - index_ = (*cur) & 0x7f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return finish_indexed_field(cur + 1, end); -} - -/* parse an indexed field with index >= 127 */ -grpc_error_handle HPackParser::parse_indexed_field_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = {&HPackParser::finish_indexed_field}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0x7f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* When finishing with a header, get the cached md element for this index. - This is set in parse_value_string(). We ensure (in debug mode) that the - cached metadata corresponds with the index we are examining. */ -grpc_mdelem HPackParser::GetPrecomputedMDForIndex() { - GPR_DEBUG_ASSERT(md_for_index_.payload != 0); - GPR_DEBUG_ASSERT(static_cast(index_) == precomputed_md_index_); - grpc_mdelem md = md_for_index_; - GPR_DEBUG_ASSERT(!GRPC_MDISNULL(md)); /* handled in string parsing */ - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ -#ifndef NDEBUG - precomputed_md_index_ = -1; -#endif - return md; -} - -static const grpc_core::ManagedMemorySlice& get_indexed_key(grpc_mdelem md) { - GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(md)); - return static_cast( - grpc_slice_ref_internal(GRPC_MDKEY(md))); -} - -/* finish a literal header with incremental indexing */ -grpc_error_handle HPackParser::finish_lithdr_incidx(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX(); - grpc_mdelem md = GetPrecomputedMDForIndex(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(get_indexed_key(md), value_.TakeIntern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* finish a literal header with incremental indexing with no index */ -grpc_error_handle HPackParser::finish_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_INCIDX_V(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(key_.TakeIntern(), value_.TakeIntern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a literal header with incremental indexing; index < 63 */ -grpc_error_handle HPackParser::parse_lithdr_incidx(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_incidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = (*cur) & 0x3f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return parse_string_prefix(cur + 1, end); -} - -/* parse a literal header with incremental indexing; index >= 63 */ -grpc_error_handle HPackParser::parse_lithdr_incidx_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_incidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0x3f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* parse a literal header with incremental indexing; index = 0 */ -grpc_error_handle HPackParser::parse_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_key_string, &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_literal_key, - &HPackParser::finish_lithdr_incidx_v}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - return parse_string_prefix(cur + 1, end); -} - -/* finish a literal header without incremental indexing */ -grpc_error_handle HPackParser::finish_lithdr_notidx(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX(); - grpc_mdelem md = GetPrecomputedMDForIndex(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(get_indexed_key(md), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* finish a literal header without incremental indexing with index = 0 */ -grpc_error_handle HPackParser::finish_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NOTIDX_V(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(key_.TakeIntern(), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a literal header without incremental indexing; index < 15 */ -grpc_error_handle HPackParser::parse_lithdr_notidx(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_notidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = (*cur) & 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return parse_string_prefix(cur + 1, end); -} - -/* parse a literal header without incremental indexing; index >= 15 */ -grpc_error_handle HPackParser::parse_lithdr_notidx_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_notidx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* parse a literal header without incremental indexing; index == 0 */ -grpc_error_handle HPackParser::parse_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_key_string, &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_literal_key, - &HPackParser::finish_lithdr_notidx_v}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - return parse_string_prefix(cur + 1, end); -} - -/* finish a literal header that is never indexed */ -grpc_error_handle HPackParser::finish_lithdr_nvridx(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX(); - grpc_mdelem md = GetPrecomputedMDForIndex(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(get_indexed_key(md), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* finish a literal header that is never indexed with an extra value */ -grpc_error_handle HPackParser::finish_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end) { - GRPC_STATS_INC_HPACK_RECV_LITHDR_NVRIDX_V(); - grpc_error_handle err = FinishHeader( - grpc_mdelem_from_slices(key_.TakeIntern(), value_.TakeExtern())); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} - -/* parse a literal header that is never indexed; index < 15 */ -grpc_error_handle HPackParser::parse_lithdr_nvridx(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_nvridx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = (*cur) & 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return parse_string_prefix(cur + 1, end); -} - -/* parse a literal header that is never indexed; index >= 15 */ -grpc_error_handle HPackParser::parse_lithdr_nvridx_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_indexed_key, - &HPackParser::finish_lithdr_nvridx}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - index_ = 0xf; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} - -/* parse a literal header that is never indexed; index == 0 */ -grpc_error_handle HPackParser::parse_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = { - &HPackParser::parse_key_string, &HPackParser::parse_string_prefix, - &HPackParser::parse_value_string_with_literal_key, - &HPackParser::finish_lithdr_nvridx_v}; - dynamic_table_updates_allowed_ = 0; - next_state_ = and_then; - return parse_string_prefix(cur + 1, end); -} - -/* finish parsing a max table size change */ -grpc_error_handle HPackParser::finish_max_tbl_size(const uint8_t* cur, - const uint8_t* end) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { - gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", index_); + String& operator=(String&& other) noexcept { + value_ = std::move(other.value_); + other.value_ = absl::Span(); + return *this; } - grpc_error_handle err = - grpc_chttp2_hptbl_set_current_table_size(&table_, index_); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_begin(cur, end); -} -/* parse a max table size change, max size < 15 */ -grpc_error_handle HPackParser::parse_max_tbl_size(const uint8_t* cur, - const uint8_t* end) { - if (dynamic_table_updates_allowed_ == 0) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "More than two max table size changes in a single frame")); + // Parse a non-binary string + static absl::optional Parse(Input* input) { + auto pfx = input->ParseStringPrefix(); + if (!pfx.has_value()) return {}; + if (pfx->huff) { + // Huffman coded + std::vector output; + auto v = ParseHuff(input, pfx->length, + [&output](uint8_t c) { output.push_back(c); }); + if (!v) return {}; + return String(std::move(output)); + } + return ParseUncompressed(input, pfx->length); } - dynamic_table_updates_allowed_--; - index_ = (*cur) & 0x1f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - return finish_max_tbl_size(cur + 1, end); -} -/* parse a max table size change, max size >= 15 */ -grpc_error_handle HPackParser::parse_max_tbl_size_x(const uint8_t* cur, - const uint8_t* end) { - static const State and_then[] = {&HPackParser::finish_max_tbl_size}; - if (dynamic_table_updates_allowed_ == 0) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "More than two max table size changes in a single frame")); + // Parse a binary string + static absl::optional ParseBinary(Input* input) { + auto pfx = input->ParseStringPrefix(); + if (!pfx.has_value()) return {}; + if (!pfx->huff) { + if (pfx->length > 0 && input->peek() == 0) { + // 'true-binary' + input->Advance(1); + return ParseUncompressed(input, pfx->length - 1); + } + // Base64 encoded... pull out the string, then unbase64 it + auto base64 = ParseUncompressed(input, pfx->length); + if (!base64.has_value()) return {}; + return Unbase64(input, std::move(*base64)); + } else { + // Huffman encoded... + std::vector decompressed; + // State here says either we don't know if it's base64 or binary, or we do + // and what is it. + enum class State { kUnsure, kBinary, kBase64 }; + State state = State::kUnsure; + auto decompressed_ok = + ParseHuff(input, pfx->length, [&state, &decompressed](uint8_t c) { + if (state == State::kUnsure) { + // First byte... if it's zero it's binary + if (c == 0) { + // Save the type, and skip the zero + state = State::kBinary; + return; + } else { + // Flag base64, store this value + state = State::kBase64; + } + } + // Non-first byte, or base64 first byte + decompressed.push_back(c); + }); + if (!decompressed_ok) return {}; + switch (state) { + case State::kUnsure: + // No bytes, empty span + return String(absl::Span()); + case State::kBinary: + // Binary, we're done + return String(std::move(decompressed)); + case State::kBase64: + // Base64 - unpack it + return Unbase64(input, String(std::move(decompressed))); + } + GPR_UNREACHABLE_CODE(abort();); + } } - dynamic_table_updates_allowed_--; - next_state_ = and_then; - index_ = 0x1f; - md_for_index_.payload = 0; /* Invalidate cached md when index changes. */ - parsing_.value = &index_; - return parse_value0(cur + 1, end); -} -/* a parse error: jam the parse state into parse_error, and return error */ -grpc_error_handle HPackParser::parse_error(const uint8_t* /*cur*/, - const uint8_t* /*end*/, - grpc_error_handle err) { - GPR_ASSERT(err != GRPC_ERROR_NONE); - if (last_error_ == GRPC_ERROR_NONE) { - last_error_ = GRPC_ERROR_REF(err); + private: + void AppendBytes(const uint8_t* data, size_t length); + explicit String(std::vector v) : value_(std::move(v)) {} + explicit String(absl::Span v) : value_(v) {} + String(grpc_slice_refcount* r, const uint8_t* begin, const uint8_t* end) + : value_(MakeSlice(r, begin, end)) {} + + // Given a refcount and a byte range, make a slice + static grpc_slice MakeSlice(grpc_slice_refcount* r, const uint8_t* begin, + const uint8_t* end) { + grpc_slice out; + out.refcount = r; + r->Ref(); + out.data.refcounted.bytes = const_cast(begin); + out.data.refcounted.length = end - begin; + return out; } - state_ = &HPackParser::still_parse_error; - return err; -} - -grpc_error_handle HPackParser::still_parse_error(const uint8_t* /*cur*/, - const uint8_t* /*end*/) { - return GRPC_ERROR_REF(last_error_); -} -grpc_error_handle HPackParser::parse_illegal_op(const uint8_t* cur, - const uint8_t* end) { - GPR_ASSERT(cur != end); - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("Illegal hpack op code ", *cur).c_str()); - return parse_error(cur, end, err); -} - -/* parse the 1st byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value0(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value0; - return GRPC_ERROR_NONE; + // Parse some huffman encoded bytes, using output(uint8_t b) to emit each + // decoded byte. + template + static bool ParseHuff(Input* input, uint32_t length, Out output) { + GRPC_STATS_INC_HPACK_RECV_HUFFMAN(); + int16_t state = 0; + // Parse one half byte... we leverage some lookup tables to keep the logic + // here really simple. + auto nibble = [&output, &state](uint8_t nibble) { + int16_t emit = emit_sub_tbl[16 * emit_tbl[state] + nibble]; + int16_t next = next_sub_tbl[16 * next_tbl[state] + nibble]; + if (emit != -1) { + if (emit >= 0 && emit < 256) { + output(static_cast(emit)); + } else { + assert(emit == 256); + } + } + state = next; + }; + // If there's insufficient bytes remaining, return now. + if (input->remaining() < length) { + return input->UnexpectedEOF(false); + } + // Grab the byte range, and iterate through it. + const uint8_t* p = input->cur_ptr(); + input->Advance(length); + for (uint32_t i = 0; i < length; i++) { + nibble(p[i] >> 4); + nibble(p[i] & 0xf); + } + return true; } - *parsing_.value += (*cur) & 0x7f; - - if ((*cur) & 0x80) { - return parse_value1(cur + 1, end); - } else { - return parse_next(cur + 1, end); + // Parse some uncompressed string bytes. + static absl::optional ParseUncompressed(Input* input, + uint32_t length) { + GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(); + // Check there's enough bytes + if (input->remaining() < length) { + return input->UnexpectedEOF(absl::optional()); + } + auto* refcount = input->slice_refcount(); + auto* p = input->cur_ptr(); + input->Advance(length); + if (refcount != nullptr) { + return String(refcount, p, p + length); + } else { + return String(absl::Span(p, length)); + } } -} -/* parse the 2nd byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value1(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value1; - return GRPC_ERROR_NONE; + // Turn base64 encoded bytes into not base64 encoded bytes. + // Only takes input to set an error on failure. + static absl::optional Unbase64(Input* input, String s) { + auto v = Match( + s.value_, + [](const grpc_slice& slice) { + return Unbase64Loop(GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)); + }, + [](absl::Span span) { + return Unbase64Loop(span.begin(), span.end()); + }, + [](const std::vector& vec) { + return Unbase64Loop(vec.data(), vec.data() + vec.size()); + }); + if (!v.has_value()) { + return input->MaybeSetErrorAndReturn( + [] { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "illegal base64 encoding"); + }, + absl::optional()); + } + return String(std::move(*v)); } - *parsing_.value += ((static_cast(*cur)) & 0x7f) << 7; - - if ((*cur) & 0x80) { - return parse_value2(cur + 1, end); - } else { - return parse_next(cur + 1, end); - } -} + // Main loop for Unbase64 + static absl::optional> Unbase64Loop(const uint8_t* cur, + const uint8_t* end) { + while (cur != end && end[-1] == '=') { + --end; + } -/* parse the 3rd byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value2(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value2; - return GRPC_ERROR_NONE; - } + std::vector out; + out.reserve(3 * (end - cur) / 4 + 3); - *parsing_.value += ((static_cast(*cur)) & 0x7f) << 14; + // Decode 4 bytes at a time while we can + while (end - cur >= 4) { + uint32_t bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + uint32_t buffer = bits << 18; + ++cur; - if ((*cur) & 0x80) { - return parse_value3(cur + 1, end); - } else { - return parse_next(cur + 1, end); - } -} + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 12; + ++cur; -/* parse the 4th byte of a varint into parsing_.value - no overflow is possible */ -grpc_error_handle HPackParser::parse_value3(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_value3; - return GRPC_ERROR_NONE; - } + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 6; + ++cur; - *parsing_.value += ((static_cast(*cur)) & 0x7f) << 21; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits; + ++cur; - if ((*cur) & 0x80) { - return parse_value4(cur + 1, end); - } else { - return parse_next(cur + 1, end); - } -} + out.insert(out.end(), {static_cast(buffer >> 16), + static_cast(buffer >> 8), + static_cast(buffer)}); + } + // Deal with the last 0, 1, 2, or 3 bytes. + switch (end - cur) { + case 0: + return out; + case 1: + return {}; + case 2: { + uint32_t bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + uint32_t buffer = bits << 18; -/* parse the 5th byte of a varint into parsing_.value - depending on the byte, we may overflow, and care must be taken */ -grpc_error_handle HPackParser::parse_value4(const uint8_t* cur, - const uint8_t* end) { - uint8_t c; - uint32_t cur_value; - uint32_t add_value; + ++cur; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 12; - if (cur == end) { - state_ = &HPackParser::parse_value4; - return GRPC_ERROR_NONE; - } + if (buffer & 0xffff) return {}; + out.push_back(static_cast(buffer >> 16)); + return out; + } + case 3: { + uint32_t bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + uint32_t buffer = bits << 18; - c = (*cur) & 0x7f; - if (c > 0xf) { - goto error; - } + ++cur; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 12; - cur_value = *parsing_.value; - add_value = (static_cast(c)) << 28; - if (add_value > 0xffffffffu - cur_value) { - goto error; - } + ++cur; + bits = kBase64InverseTable.table[*cur]; + if (bits > 63) return {}; + buffer |= bits << 6; - *parsing_.value = cur_value + add_value; + ++cur; + if (buffer & 0xff) return {}; + out.push_back(static_cast(buffer >> 16)); + out.push_back(static_cast(buffer >> 8)); + return out; + } + } - if ((*cur) & 0x80) { - return parse_value5up(cur + 1, end); - } else { - return parse_next(cur + 1, end); + GPR_UNREACHABLE_CODE(return out;); } -error: - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat( - "integer overflow in hpack integer decoding: have 0x%08x, " - "got byte 0x%02x on byte 5", - *parsing_.value, *cur) - .c_str()); - return parse_error(cur, end, err); -} - -/* parse any trailing bytes in a varint: it's possible to append an arbitrary - number of 0x80's and not affect the value - a zero will terminate - and - anything else will overflow */ -grpc_error_handle HPackParser::parse_value5up(const uint8_t* cur, - const uint8_t* end) { - while (cur != end && *cur == 0x80) { - ++cur; - } + absl::variant, std::vector> + value_; +}; - if (cur == end) { - state_ = &HPackParser::parse_value5up; - return GRPC_ERROR_NONE; +// Parser parses one frame + continuations worth of headers. +class HPackParser::Parser { + public: + Parser(Input* input, HPackParser::Sink* sink, HPackTable* table, + uint8_t* dynamic_table_updates_allowed) + : input_(input), + sink_(sink), + table_(table), + dynamic_table_updates_allowed_(dynamic_table_updates_allowed) {} + + // Skip any priority bits, or return false on failure + bool SkipPriority() { + if (input_->remaining() < 5) return input_->UnexpectedEOF(false); + input_->Advance(5); + return true; } - if (*cur == 0) { - return parse_next(cur + 1, end); + bool Parse() { + auto cur = *input_->Next(); + switch (cur >> 4) { + // Literal header not indexed - First byte format: 0000xxxx + // Literal header never indexed - First byte format: 0001xxxx + // Where xxxx: + // 0000 - literal key + // 1111 - indexed key, varint encoded index + // other - indexed key, inline encoded index + case 0: + case 1: + switch (cur & 0xf) { + case 0: // literal key + return FinishHeader( + ParseLiteralKey()); + case 0xf: // varint encoded key index + return FinishHeader( + ParseVarIdxKey(0xf)); + default: // inline encoded key index + return FinishHeader( + ParseIdxKey(cur & 0xf)); + } + // Update max table size. + // First byte format: 001xxxxx + // Where xxxxx: + // 11111 - max size is varint encoded + // other - max size is stored inline + case 2: + // inline encoded max table size + return FinishMaxTableSize(cur & 0x1f); + case 3: + if (cur == 0x3f) { + // varint encoded max table size + return FinishMaxTableSize(input_->ParseVarint(0x1f)); + } else { + // inline encoded max table size + return FinishMaxTableSize(cur & 0x1f); + } + // Literal header with incremental indexing. + // First byte format: 01xxxxxx + // Where xxxxxx: + // 000000 - literal key + // 111111 - indexed key, varint encoded index + // other - indexed key, inline encoded index + case 4: + if (cur == 0x40) { + // literal key + return FinishHeader( + ParseLiteralKey()); + } + ABSL_FALLTHROUGH_INTENDED; + case 5: + case 6: + // inline encoded key index + return FinishHeader( + ParseIdxKey(cur & 0x3f)); + case 7: + if (cur == 0x7f) { + // varint encoded key index + return FinishHeader( + ParseVarIdxKey(0x3f)); + } else { + // inline encoded key index + return FinishHeader( + ParseIdxKey(cur & 0x3f)); + } + // Indexed Header Field Representation + // First byte format: 1xxxxxxx + // Where xxxxxxx: + // 0000000 - illegal + // 1111111 - varint encoded field index + // other - inline encoded field index + case 8: + if (cur == 0x80) { + // illegal value. + return input_->MaybeSetErrorAndReturn( + [] { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Illegal hpack op code"); + }, + false); + } + ABSL_FALLTHROUGH_INTENDED; + case 9: + case 10: + case 11: + case 12: + case 13: + case 14: + // inline encoded field index + return FinishIndexed(cur & 0x7f); + case 15: + if (cur == 0xff) { + // varint encoded field index + return FinishIndexed(input_->ParseVarint(0x7f)); + } else { + // inline encoded field index + return FinishIndexed(cur & 0x7f); + } + } + GPR_UNREACHABLE_CODE(abort()); } - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat( - "integer overflow in hpack integer decoding: have 0x%08x, " - "got byte 0x%02x sometime after byte 5", - *parsing_.value, *cur) - .c_str()); - return parse_error(cur, end, err); -} - -/* parse a string prefix */ -grpc_error_handle HPackParser::parse_string_prefix(const uint8_t* cur, - const uint8_t* end) { - if (cur == end) { - state_ = &HPackParser::parse_string_prefix; - return GRPC_ERROR_NONE; + private: + void GPR_ATTRIBUTE_NOINLINE LogHeader(grpc_mdelem md) { + char* k = grpc_slice_to_c_string(GRPC_MDKEY(md)); + char* v = nullptr; + if (grpc_is_binary_header_internal(GRPC_MDKEY(md))) { + v = grpc_dump_slice(GRPC_MDVALUE(md), GPR_DUMP_HEX); + } else { + v = grpc_slice_to_c_string(GRPC_MDVALUE(md)); + } + gpr_log( + GPR_INFO, + "Decode: '%s: %s', elem_interned=%d [%d], k_interned=%d, v_interned=%d", + k, v, GRPC_MDELEM_IS_INTERNED(md), GRPC_MDELEM_STORAGE(md), + grpc_slice_is_interned(GRPC_MDKEY(md)), + grpc_slice_is_interned(GRPC_MDVALUE(md))); + gpr_free(k); + gpr_free(v); } - strlen_ = (*cur) & 0x7f; - huff_ = (*cur) >> 7; - if (strlen_ == 0x7f) { - parsing_.value = &strlen_; - return parse_value0(cur + 1, end); - } else { - return parse_next(cur + 1, end); + // During FinishHeader, how should the header be treated in the hpack table + enum class TableAction { + // Add to the table + kAddToTable, + // Do not add to the table + kOmitFromTable, + }; + + template + bool FinishHeader(grpc_mdelem md) { + // Allow higher code to just pass in failures ... simplifies things a bit. + if (GRPC_MDISNULL(md)) return false; + // Log if desired + if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { + LogHeader(md); + } + // Add to the hpack table if needed + if (action == TableAction::kAddToTable) { + GPR_DEBUG_ASSERT(GRPC_MDELEM_STORAGE(md) == + GRPC_MDELEM_STORAGE_INTERNED || + GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC); + grpc_error_handle err = table_->Add(md); + if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) { + input_->SetError(err); + return false; + }; + } + // Pass up to the transport + grpc_error_handle err = (*sink_)(md); + if (GPR_UNLIKELY(err != GRPC_ERROR_NONE)) { + input_->SetError(err); + return false; + } + return true; } -} -/* append some bytes to a string */ -void HPackParser::String::AppendBytes(const uint8_t* data, size_t length) { - if (length == 0) return; - if (length + data_.copied.length > data_.copied.capacity) { - GPR_ASSERT(data_.copied.length + length <= UINT32_MAX); - data_.copied.capacity = static_cast(data_.copied.length + length); - data_.copied.str = static_cast( - gpr_realloc(data_.copied.str, data_.copied.capacity)); + // Parse a string encoded key and a string encoded value + template + grpc_mdelem ParseLiteralKey() { + auto key = String::Parse(input_); + if (!key.has_value()) return GRPC_MDNULL; + auto key_slice = key->Take(); + auto value = ParseValueString(key_slice); + if (GPR_UNLIKELY(!value.has_value())) { + grpc_slice_unref_internal(key_slice); + return GRPC_MDNULL; + } + return grpc_mdelem_from_slices(key_slice, value->Take()); } - memcpy(data_.copied.str + data_.copied.length, data, length); - GPR_ASSERT(length <= UINT32_MAX - data_.copied.length); - data_.copied.length += static_cast(length); -} -grpc_error_handle HPackParser::AppendString(const uint8_t* cur, - const uint8_t* end) { - String* str = parsing_.str; - uint32_t bits; - uint8_t decoded[3]; - switch (binary_) { - case BinaryState::kNotBinary: - str->AppendBytes(cur, static_cast(end - cur)); - return GRPC_ERROR_NONE; - case BinaryState::kBinaryBegin: - if (cur == end) { - binary_ = BinaryState::kBinaryBegin; - return GRPC_ERROR_NONE; - } - if (*cur == 0) { - /* 'true-binary' case */ - ++cur; - binary_ = BinaryState::kNotBinary; - GRPC_STATS_INC_HPACK_RECV_BINARY(); - str->AppendBytes(cur, static_cast(end - cur)); - return GRPC_ERROR_NONE; - } - GRPC_STATS_INC_HPACK_RECV_BINARY_BASE64(); - b64_byte0: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte0: - if (cur == end) { - binary_ = BinaryState::kBase64Byte0; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte0; - } - base64_buffer_ = bits << 18; - b64_byte1: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte1: - if (cur == end) { - binary_ = BinaryState::kBase64Byte1; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte1; - } - base64_buffer_ |= bits << 12; - b64_byte2: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte2: - if (cur == end) { - binary_ = BinaryState::kBase64Byte2; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte2; - } - base64_buffer_ |= bits << 6; - b64_byte3: - ABSL_FALLTHROUGH_INTENDED; - case BinaryState::kBase64Byte3: - if (cur == end) { - binary_ = BinaryState::kBase64Byte3; - return GRPC_ERROR_NONE; - } - bits = kBase64InverseTable.table[*cur]; - ++cur; - if (bits == 255) { - return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Illegal base64 character")); - } else if (bits == 64) { - goto b64_byte3; - } - base64_buffer_ |= bits; - bits = base64_buffer_; - decoded[0] = static_cast(bits >> 16); - decoded[1] = static_cast(bits >> 8); - decoded[2] = static_cast(bits); - str->AppendBytes(decoded, 3); - goto b64_byte0; + // Parse an index encoded key and a string encoded value + template + grpc_mdelem ParseIdxKey(uint32_t index) { + auto elem = table_->Peek(index); + if (GPR_UNLIKELY(GRPC_MDISNULL(elem))) { + return InvalidHPackIndexError(index, elem); + } + GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(elem)); + auto value = ParseValueString(GRPC_MDKEY(elem)); + if (GPR_UNLIKELY(!value.has_value())) return GRPC_MDNULL; + return grpc_mdelem_from_slices( + static_cast( + grpc_slice_ref_internal(GRPC_MDKEY(elem))), + value->Take()); } - GPR_UNREACHABLE_CODE(return parse_error( - cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING("Should never reach here"))); -} -grpc_error_handle HPackParser::finish_str(const uint8_t* cur, - const uint8_t* end) { - uint8_t decoded[2]; - uint32_t bits; - String* str = parsing_.str; - switch (binary_) { - case BinaryState::kNotBinary: - break; - case BinaryState::kBinaryBegin: - break; - case BinaryState::kBase64Byte0: - break; - case BinaryState::kBase64Byte1: - return parse_error(cur, end, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "illegal base64 encoding")); /* illegal encoding */ - case BinaryState::kBase64Byte2: - bits = base64_buffer_; - if (bits & 0xffff) { - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat("trailing bits in base64 encoding: 0x%04x", - bits & 0xffff) - .c_str()); - return parse_error(cur, end, err); - } - decoded[0] = static_cast(bits >> 16); - str->AppendBytes(decoded, 1); - break; - case BinaryState::kBase64Byte3: - bits = base64_buffer_; - if (bits & 0xff) { - grpc_error_handle err = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrFormat("trailing bits in base64 encoding: 0x%02x", - bits & 0xff) - .c_str()); - return parse_error(cur, end, err); - } - decoded[0] = static_cast(bits >> 16); - decoded[1] = static_cast(bits >> 8); - str->AppendBytes(decoded, 2); - break; + // Parse a varint index encoded key and a string encoded value + template + grpc_mdelem ParseVarIdxKey(uint32_t offset) { + auto index = input_->ParseVarint(offset); + if (GPR_UNLIKELY(!index.has_value())) return GRPC_MDNULL; + return ParseIdxKey(*index); } - return GRPC_ERROR_NONE; -} -/* decode a nibble from a huffman encoded stream */ -grpc_error_handle HPackParser::AppendHuffNibble(uint8_t nibble) { - int16_t emit = emit_sub_tbl[16 * emit_tbl[huff_state_] + nibble]; - int16_t next = next_sub_tbl[16 * next_tbl[huff_state_] + nibble]; - if (emit != -1) { - if (emit >= 0 && emit < 256) { - uint8_t c = static_cast(emit); - grpc_error_handle err = AppendString(&c, (&c) + 1); - if (err != GRPC_ERROR_NONE) return err; + // Parse a string, figuring out if it's binary or not by the key name. + template + absl::optional ParseValueString(const SliceType& key) { + if (grpc_is_refcounted_slice_binary_header(key)) { + return String::ParseBinary(input_); } else { - assert(emit == 256); + return String::Parse(input_); } } - huff_state_ = next; - return GRPC_ERROR_NONE; -} -/* decode full bytes from a huffman encoded stream */ -grpc_error_handle HPackParser::AppendHuffBytes(const uint8_t* cur, - const uint8_t* end) { - for (; cur != end; ++cur) { - grpc_error_handle err = AppendHuffNibble(*cur >> 4); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - err = AppendHuffNibble(*cur & 0xf); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); + // Emit an indexed field + bool FinishIndexed(absl::optional index) { + *dynamic_table_updates_allowed_ = 0; + if (!index.has_value()) return false; + grpc_mdelem md = table_->Fetch(*index); + if (GPR_UNLIKELY(GRPC_MDISNULL(md))) { + return InvalidHPackIndexError(*index, false); + } + GRPC_STATS_INC_HPACK_RECV_INDEXED(); + return FinishHeader(md); } - return GRPC_ERROR_NONE; -} -/* decode some string bytes based on the current decoding mode - (huffman or not) */ -grpc_error_handle HPackParser::AppendStrBytes(const uint8_t* cur, - const uint8_t* end) { - if (huff_) { - return AppendHuffBytes(cur, end); - } else { - return AppendString(cur, end); + // finish parsing a max table size change + bool FinishMaxTableSize(absl::optional size) { + if (!size.has_value()) return false; + if (*dynamic_table_updates_allowed_ == 0) { + return input_->MaybeSetErrorAndReturn( + [] { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "More than two max table size changes in a single frame"); + }, + false); + } + (*dynamic_table_updates_allowed_)--; + if (GRPC_TRACE_FLAG_ENABLED(grpc_trace_chttp2_hpack_parser)) { + gpr_log(GPR_INFO, "MAX TABLE SIZE: %d", *size); + } + grpc_error_handle err = table_->SetCurrentTableSize(*size); + if (err != GRPC_ERROR_NONE) { + input_->SetError(err); + return false; + } + return true; } -} -/* parse a string - tries to do large chunks at a time */ -grpc_error_handle HPackParser::parse_string(const uint8_t* cur, - const uint8_t* end) { - size_t remaining = strlen_ - strgot_; - size_t given = static_cast(end - cur); - if (remaining <= given) { - grpc_error_handle err = AppendStrBytes(cur, cur + remaining); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - err = finish_str(cur + remaining, end); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_next(cur + remaining, end); - } else { - grpc_error_handle err = AppendStrBytes(cur, cur + given); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - GPR_ASSERT(given <= UINT32_MAX - strgot_); - strgot_ += static_cast(given); - state_ = &HPackParser::parse_string; - return GRPC_ERROR_NONE; + // Set an invalid hpack index error if no error has been set. Returns result + // unmodified. + template + R InvalidHPackIndexError(uint32_t index, R result) { + return input_->MaybeSetErrorAndReturn( + [this, index] { + return grpc_error_set_int( + grpc_error_set_int(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Invalid HPACK index received"), + GRPC_ERROR_INT_INDEX, + static_cast(index)), + GRPC_ERROR_INT_SIZE, + static_cast(this->table_->num_entries())); + }, + result); } -} -/* begin parsing a string - performs setup, calls parse_string */ -grpc_error_handle HPackParser::begin_parse_string(const uint8_t* cur, - const uint8_t* end, - BinaryState binary, - HPackParser::String* str) { - if (!huff_ && binary == BinaryState::kNotBinary && - static_cast(end - cur) >= strlen_ && - current_slice_refcount_ != nullptr) { - GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(); - str->copied_ = false; - str->data_.referenced.refcount = current_slice_refcount_; - str->data_.referenced.data.refcounted.bytes = const_cast(cur); - str->data_.referenced.data.refcounted.length = strlen_; - grpc_slice_ref_internal(str->data_.referenced); - return parse_next(cur + strlen_, end); - } - strgot_ = 0; - str->copied_ = true; - str->data_.copied.length = 0; - parsing_.str = str; - huff_state_ = 0; - binary_ = binary; - switch (binary_) { - case BinaryState::kNotBinary: - if (huff_) { - GRPC_STATS_INC_HPACK_RECV_HUFFMAN(); - } else { - GRPC_STATS_INC_HPACK_RECV_UNCOMPRESSED(); - } - break; - case BinaryState::kBinaryBegin: - /* stats incremented later: don't know true binary or not */ - break; - default: - abort(); - } - return parse_string(cur, end); -} + Input* input_; + HPackParser::Sink* sink_; + HPackTable* const table_; + uint8_t* dynamic_table_updates_allowed_; +}; -/* parse the key string */ -grpc_error_handle HPackParser::parse_key_string(const uint8_t* cur, - const uint8_t* end) { - return begin_parse_string(cur, end, BinaryState::kNotBinary, &key_); +UnmanagedMemorySlice HPackParser::String::Take(Extern) { + auto s = Match( + value_, + [](const grpc_slice& slice) { + GPR_DEBUG_ASSERT(!grpc_slice_is_interned(slice)); + return static_cast(slice); + }, + [](absl::Span span) { + return UnmanagedMemorySlice( + reinterpret_cast(const_cast(span.begin())), + span.size()); + }, + [](const std::vector& v) { + return UnmanagedMemorySlice(reinterpret_cast(v.data()), + v.size()); + }); + value_ = absl::Span(); + return s; } -/* check if a key represents a binary header or not */ - -bool HPackParser::IsBinaryLiteralHeader() { - /* We know that either argument here is a reference counter slice. - * 1. If it is a grpc_core::StaticSlice, the refcount is set to kNoopRefcount. - * 2. If it's key_.data.referenced, then key_.copied was set to false, - * which occurs in begin_parse_string() - where the refcount is set to - * current_slice_refcount_, which is not null. */ - return grpc_is_refcounted_slice_binary_header( - key_.copied_ ? grpc_core::ExternallyManagedSlice(key_.data_.copied.str, - key_.data_.copied.length) - : key_.data_.referenced); +ManagedMemorySlice HPackParser::String::Take(Intern) { + auto s = Match( + value_, + [](const grpc_slice& slice) { + ManagedMemorySlice s(&slice); + grpc_slice_unref_internal(slice); + return s; + }, + [](absl::Span span) { + return ManagedMemorySlice( + reinterpret_cast(const_cast(span.data())), + span.size()); + }, + [](const std::vector& v) { + return ManagedMemorySlice(reinterpret_cast(v.data()), + v.size()); + }); + value_ = absl::Span(); + return s; } -/* Cache the metadata for the given index during initial parsing. This avoids a - pointless recomputation of the metadata when finishing a header. We read the - cached value in get_precomputed_md_for_idx(). */ -void HPackParser::SetPrecomputedMDIndex(grpc_mdelem md) { - GPR_DEBUG_ASSERT(md_for_index_.payload == 0); - GPR_DEBUG_ASSERT(precomputed_md_index_ == -1); - md_for_index_ = md; -#ifndef NDEBUG - precomputed_md_index_ = index_; -#endif -} +/* PUBLIC INTERFACE */ -/* Determines if a metadata element key associated with the current parser index - is a binary indexed header during string parsing. We'll need to revisit this - metadata when we're done parsing, so we cache the metadata for this index - here using set_precomputed_md_idx(). */ -grpc_error_handle HPackParser::IsBinaryIndexedHeader(bool* is) { - grpc_mdelem elem = grpc_chttp2_hptbl_lookup(&table_, index_); - if (GPR_UNLIKELY(GRPC_MDISNULL(elem))) { - return InvalidHPackIndexError(); - } - /* We know that GRPC_MDKEY(elem) points to a reference counted slice since: - * 1. elem was a result of grpc_chttp2_hptbl_lookup - * 2. An item in this table is either static (see entries with - * index < GRPC_CHTTP2_LAST_STATIC_ENTRY or added via - * grpc_chttp2_hptbl_add). - * 3. If added via grpc_chttp2_hptbl_add, the entry is either static or - * interned. - * 4. Both static and interned element slices have non-null refcounts. */ - *is = grpc_is_refcounted_slice_binary_header(GRPC_MDKEY(elem)); - SetPrecomputedMDIndex(elem); - return GRPC_ERROR_NONE; -} +HPackParser::HPackParser() = default; -/* parse the value string */ -grpc_error_handle HPackParser::parse_value_string(const uint8_t* cur, - const uint8_t* end, - bool is_binary) { - return begin_parse_string( - cur, end, is_binary ? BinaryState::kBinaryBegin : BinaryState::kNotBinary, - &value_); -} +HPackParser::~HPackParser() = default; -grpc_error_handle HPackParser::parse_value_string_with_indexed_key( - const uint8_t* cur, const uint8_t* end) { - bool is_binary = false; - grpc_error_handle err = IsBinaryIndexedHeader(&is_binary); - if (err != GRPC_ERROR_NONE) return parse_error(cur, end, err); - return parse_value_string(cur, end, is_binary); +void HPackParser::BeginFrame(Sink sink, Boundary boundary, Priority priority) { + sink_ = std::move(sink); + boundary_ = boundary; + priority_ = priority; + dynamic_table_updates_allowed_ = 2; } -grpc_error_handle HPackParser::parse_value_string_with_literal_key( - const uint8_t* cur, const uint8_t* end) { - return parse_value_string(cur, end, IsBinaryLiteralHeader()); +grpc_error_handle HPackParser::Parse(const grpc_slice& slice, bool is_last) { + if (GPR_UNLIKELY(!unparsed_bytes_.empty())) { + std::vector buffer = std::move(unparsed_bytes_); + buffer.insert(buffer.end(), GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)); + return ParseInput( + Input(nullptr, buffer.data(), buffer.data() + buffer.size()), is_last); + } + return ParseInput(Input(slice.refcount, GRPC_SLICE_START_PTR(slice), + GRPC_SLICE_END_PTR(slice)), + is_last); } -/* PUBLIC INTERFACE */ - -HPackParser::HPackParser() { - state_ = &HPackParser::parse_begin; - key_.data_.referenced = grpc_empty_slice(); - key_.data_.copied.str = nullptr; - key_.data_.copied.capacity = 0; - key_.data_.copied.length = 0; - value_.data_.referenced = grpc_empty_slice(); - value_.data_.copied.str = nullptr; - value_.data_.copied.capacity = 0; - value_.data_.copied.length = 0; - /* Cached metadata for the current index the parser is handling. This is set - to 0 initially, invalidated when the index changes, and invalidated when it - is read (by get_precomputed_md_for_idx()). It is set during string parsing, - by set_precomputed_md_idx() - which is called by parse_value_string(). - The goal here is to avoid recomputing the metadata for the index when - finishing with a header as well as the initial parse. */ - md_for_index_.payload = 0; -#ifndef NDEBUG - /* In debug mode, this ensures that the cached metadata we're reading is in - * fact correct for the index we are examining. */ - precomputed_md_index_ = -1; -#endif - dynamic_table_updates_allowed_ = 2; - last_error_ = GRPC_ERROR_NONE; +grpc_error_handle HPackParser::ParseInput(Input input, bool is_last) { + if (ParseInputInner(&input)) { + return GRPC_ERROR_NONE; + } + if (input.eof_error()) { + if (GPR_UNLIKELY(is_last && is_boundary())) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Incomplete header at the end of a header/continuation sequence"); + } + unparsed_bytes_ = std::vector(input.frontier(), input.end_ptr()); + return GRPC_ERROR_NONE; + } + return input.TakeError(); } -void HPackParser::BeginFrame(Sink sink, Boundary boundary, Priority priority) { - sink_ = std::move(sink); - boundary_ = boundary; - switch (priority) { - case Priority::Included: - after_prioritization_ = state_; - state_ = &HPackParser::parse_stream_dep0; - break; +bool HPackParser::ParseInputInner(Input* input) { + switch (priority_) { case Priority::None: break; + case Priority::Included: { + if (input->remaining() < 5) return input->UnexpectedEOF(false); + input->Advance(5); + input->UpdateFrontier(); + priority_ = Priority::None; + } } -} - -HPackParser::~HPackParser() { - grpc_chttp2_hptbl_destroy(&table_); - GRPC_ERROR_UNREF(last_error_); - grpc_slice_unref_internal(key_.data_.referenced); - grpc_slice_unref_internal(value_.data_.referenced); - gpr_free(key_.data_.copied.str); - gpr_free(value_.data_.copied.str); -} - -grpc_error_handle HPackParser::Parse(const grpc_slice& slice) { -/* max number of bytes to parse at a time... limits call stack depth on - * compilers without TCO */ -#define MAX_PARSE_LENGTH 1024 - current_slice_refcount_ = slice.refcount; - const uint8_t* start = GRPC_SLICE_START_PTR(slice); - const uint8_t* end = GRPC_SLICE_END_PTR(slice); - grpc_error_handle error = GRPC_ERROR_NONE; - while (start != end && error == GRPC_ERROR_NONE) { - const uint8_t* target = start + GPR_MIN(MAX_PARSE_LENGTH, end - start); - error = (this->*state_)(start, target); - start = target; + while (!input->end_of_stream()) { + if (GPR_UNLIKELY( + !Parser(input, &sink_, &table_, &dynamic_table_updates_allowed_) + .Parse())) { + return false; + } + input->UpdateFrontier(); } - current_slice_refcount_ = nullptr; - return error; + return true; } +void HPackParser::FinishFrame() { sink_ = Sink(); } + } // namespace grpc_core // TODO(ctiller): this serves as an eviction notice for the remainder of this @@ -1630,15 +1375,11 @@ grpc_error_handle grpc_chttp2_header_parser_parse(void* hpack_parser, if (s != nullptr) { s->stats.incoming.header_bytes += GRPC_SLICE_LENGTH(slice); } - grpc_error_handle error = parser->Parse(slice); + grpc_error_handle error = parser->Parse(slice, is_last != 0); if (error != GRPC_ERROR_NONE) { return error; } if (is_last) { - if (parser->is_boundary() && !parser->is_in_begin_state()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "end of header frame not aligned with a hpack record boundary"); - } /* need to check for null stream: this can occur if we receive an invalid stream id on a header */ if (s != nullptr) { diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.h b/src/core/ext/transport/chttp2/transport/hpack_parser.h index 01fb833df1e..d6b6a49e03b 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_parser.h +++ b/src/core/ext/transport/chttp2/transport/hpack_parser.h @@ -29,10 +29,25 @@ namespace grpc_core { +// Top level interface for parsing a sequence of header, continuation frames. class HPackParser { public: - enum class Boundary { None, EndOfHeaders, EndOfStream }; - enum class Priority { None, Included }; + // What kind of stream boundary is provided by this frame? + enum class Boundary : uint8_t { + // More continuations are expected + None, + // This marks the end of headers, so data frames should follow + EndOfHeaders, + // This marks the end of headers *and* the end of the stream + EndOfStream + }; + // What kind of priority is represented in the next frame + enum class Priority : uint8_t { + // No priority field + None, + // Yes there's a priority field + Included + }; // User specified structure called for each received header. using Sink = std::function; @@ -40,189 +55,53 @@ class HPackParser { HPackParser(); ~HPackParser(); + // Non-copyable/movable HPackParser(const HPackParser&) = delete; HPackParser& operator=(const HPackParser&) = delete; + // Begin parsing a new frame + // Sink receives each parsed header, void BeginFrame(Sink sink, Boundary boundary, Priority priority); + // Change the header sink mid parse void ResetSink(Sink sink) { sink_ = std::move(sink); } - grpc_error_handle Parse(const grpc_slice& slice); + // Parse one slice worth of data + grpc_error_handle Parse(const grpc_slice& slice, bool is_last); + // Reset state ready for the next BeginFrame void FinishFrame(); - grpc_chttp2_hptbl* hpack_table() { return &table_; } + // Retrieve the associated hpack table (for tests, debugging) + HPackTable* hpack_table() { return &table_; } + // Is the current frame a boundary of some sort bool is_boundary() const { return boundary_ != Boundary::None; } + // Is the current frame the end of a stream bool is_eof() const { return boundary_ == Boundary::EndOfStream; } - bool is_in_begin_state() const { return state_ == &HPackParser::parse_begin; } private: - enum class BinaryState { - kNotBinary, - kBinaryBegin, - kBase64Byte0, - kBase64Byte1, - kBase64Byte2, - kBase64Byte3, - }; - - struct String { - bool copied_; - struct { - grpc_slice referenced; - struct { - char* str; - uint32_t length; - uint32_t capacity; - } copied; - } data_; - - UnmanagedMemorySlice TakeExtern(); - ManagedMemorySlice TakeIntern(); - void AppendBytes(const uint8_t* data, size_t length); - }; - - using State = grpc_error_handle (HPackParser::*)(const uint8_t* beg, - const uint8_t* end); - - // Forward declarations for parsing states. - // These are keeping their old (C-style) names until a future refactor where - // they will be eliminated. - grpc_error_handle parse_next(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_begin(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_error(const uint8_t* cur, const uint8_t* end, - grpc_error_handle error); - grpc_error_handle still_parse_error(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_illegal_op(const uint8_t* cur, const uint8_t* end); - - grpc_error_handle parse_string_prefix(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_key_string(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value_string(const uint8_t* cur, const uint8_t* end, - bool is_binary); - grpc_error_handle parse_value_string_with_indexed_key(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_value_string_with_literal_key(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_stream_weight(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value0(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value1(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value2(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value3(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value4(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_value5up(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep0(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep1(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep2(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_stream_dep3(const uint8_t* cur, const uint8_t* end); - - grpc_error_handle parse_indexed_field(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_indexed_field_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_incidx(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_lithdr_incidx_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_notidx(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_lithdr_notidx_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_nvridx(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_lithdr_nvridx_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_max_tbl_size(const uint8_t* cur, const uint8_t* end); - grpc_error_handle parse_max_tbl_size_x(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle parse_string(const uint8_t* cur, const uint8_t* end); - grpc_error_handle begin_parse_string(const uint8_t* cur, const uint8_t* end, - BinaryState binary, String* str); - - grpc_error_handle finish_indexed_field(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_incidx(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_incidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_notidx(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_notidx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_nvridx(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_lithdr_nvridx_v(const uint8_t* cur, - const uint8_t* end); - grpc_error_handle finish_max_tbl_size(const uint8_t* cur, const uint8_t* end); - grpc_error_handle finish_str(const uint8_t* cur, const uint8_t* end); - - enum class TableAction { - kAddToTable, - kOmitFromTable, - }; - - GPR_ATTRIBUTE_NOINLINE grpc_error_handle InvalidHPackIndexError(); - GPR_ATTRIBUTE_NOINLINE void LogHeader(grpc_mdelem md); - grpc_error_handle AddHeaderToTable(grpc_mdelem md); - template - grpc_error_handle FinishHeader(grpc_mdelem md); - - grpc_mdelem GetPrecomputedMDForIndex(); - void SetPrecomputedMDIndex(grpc_mdelem md); - bool IsBinaryLiteralHeader(); - grpc_error_handle IsBinaryIndexedHeader(bool* is); + // Helper classes: see implementation + class Parser; + class Input; + class String; - grpc_error_handle AppendString(const uint8_t* cur, const uint8_t* end); - grpc_error_handle AppendHuffNibble(uint8_t nibble); - grpc_error_handle AppendHuffBytes(const uint8_t* cur, const uint8_t* end); - grpc_error_handle AppendStrBytes(const uint8_t* cur, const uint8_t* end); + grpc_error_handle ParseInput(Input input, bool is_last); + bool ParseInputInner(Input* input); + // Callback per header received Sink sink_; - grpc_error_handle last_error_; - - // current parse state - or a function that implements it - State state_; - // future states dependent on the opening op code - const State* next_state_; - // what to do after skipping prioritization data - State after_prioritization_; - // the refcount of the slice that we're currently parsing - grpc_slice_refcount* current_slice_refcount_; - // the value we're currently parsing - union { - uint32_t* value; - String* str; - } parsing_; - // string parameters for each chunk - String key_; - String value_; - // parsed index - uint32_t index_; - // When we parse a value string, we determine the metadata element for a - // specific index, which we need again when we're finishing up with that - // header. To avoid calculating the metadata element for that index a second - // time at that stage, we cache (and invalidate) the element here. - grpc_mdelem md_for_index_; -#ifndef NDEBUG - int64_t precomputed_md_index_; -#endif - // length of source bytes for the currently parsing string - uint32_t strlen_; - // number of source bytes read for the currently parsing string - uint32_t strgot_; - // huffman decoding state - int16_t huff_state_; - // is the string being decoded binary? - BinaryState binary_; - // is the current string huffman encoded? - bool huff_; - // is a dynamic table update allowed? - uint8_t dynamic_table_updates_allowed_; - // set by higher layers, used by grpc_chttp2_header_parser_parse to signal - // it should append a metadata boundary at the end of frame + + // Bytes that could not be parsed last parsing round + std::vector unparsed_bytes_; + // Buffer kind of boundary + // TODO(ctiller): see if we can move this argument to Parse, and avoid + // buffering. Boundary boundary_; - uint32_t base64_buffer_; + // Buffer priority + // TODO(ctiller): see if we can move this argument to Parse, and avoid + // buffering. + Priority priority_; + uint8_t dynamic_table_updates_allowed_; // hpack table - grpc_chttp2_hptbl table_; + HPackTable table_; }; } // namespace grpc_core diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.cc b/src/core/ext/transport/chttp2/transport/hpack_table.cc index 696176b49d5..4e437fc982e 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.cc +++ b/src/core/ext/transport/chttp2/transport/hpack_table.cc @@ -36,191 +36,132 @@ extern grpc_core::TraceFlag grpc_http_trace; -void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl* tbl) { - size_t i; - for (i = 0; i < tbl->num_ents; i++) { - GRPC_MDELEM_UNREF(tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]); - } - gpr_free(tbl->ents); - tbl->ents = nullptr; -} +namespace grpc_core { -template -static grpc_mdelem lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index) { - /* Not static - find the value in the list of valid entries */ - tbl_index -= (GRPC_CHTTP2_LAST_STATIC_ENTRY + 1); - if (tbl_index < tbl->num_ents) { - uint32_t offset = - (tbl->num_ents - 1u - tbl_index + tbl->first_ent) % tbl->cap_entries; - grpc_mdelem md = tbl->ents[offset]; - if (take_ref) { - GRPC_MDELEM_REF(md); - } - return md; - } - /* Invalid entry: return error */ - return GRPC_MDNULL; -} +using hpack_table_detail::EntriesForBytes; +using hpack_table_detail::kInlineEntries; -grpc_mdelem grpc_chttp2_hptbl_lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index) { - return lookup_dynamic_index(tbl, tbl_index); -} +HPackTable::HPackTable() : entries_(kInlineEntries) {} -grpc_mdelem grpc_chttp2_hptbl_lookup_ref_dynamic_index( - const grpc_chttp2_hptbl* tbl, uint32_t tbl_index) { - return lookup_dynamic_index(tbl, tbl_index); +HPackTable::~HPackTable() { + for (size_t i = 0; i < num_entries_; i++) { + GRPC_MDELEM_UNREF(entries_[(first_entry_ + i) % entries_.size()]); + } } /* Evict one element from the table */ -static void evict1(grpc_chttp2_hptbl* tbl) { - grpc_mdelem first_ent = tbl->ents[tbl->first_ent]; - size_t elem_bytes = GRPC_SLICE_LENGTH(GRPC_MDKEY(first_ent)) + - GRPC_SLICE_LENGTH(GRPC_MDVALUE(first_ent)) + - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; - GPR_ASSERT(elem_bytes <= tbl->mem_used); - tbl->mem_used -= static_cast(elem_bytes); - tbl->first_ent = ((tbl->first_ent + 1) % tbl->cap_entries); - tbl->num_ents--; - GRPC_MDELEM_UNREF(first_ent); +void HPackTable::EvictOne() { + grpc_mdelem first_entry = entries_[first_entry_]; + size_t elem_bytes = GRPC_SLICE_LENGTH(GRPC_MDKEY(first_entry)) + + GRPC_SLICE_LENGTH(GRPC_MDVALUE(first_entry)) + + kEntryOverhead; + GPR_ASSERT(elem_bytes <= mem_used_); + mem_used_ -= static_cast(elem_bytes); + first_entry_ = ((first_entry_ + 1) % entries_.size()); + num_entries_--; + GRPC_MDELEM_UNREF(first_entry); } -static void rebuild_ents(grpc_chttp2_hptbl* tbl, uint32_t new_cap) { - grpc_mdelem* ents = - static_cast(gpr_malloc(sizeof(*ents) * new_cap)); - uint32_t i; - - for (i = 0; i < tbl->num_ents; i++) { - ents[i] = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; +void HPackTable::Rebuild(uint32_t new_cap) { + EntriesVec entries; + entries.resize(new_cap); + for (size_t i = 0; i < num_entries_; i++) { + entries[i] = entries_[(first_entry_ + i) % entries_.size()]; } - gpr_free(tbl->ents); - tbl->ents = ents; - tbl->cap_entries = new_cap; - tbl->first_ent = 0; + first_entry_ = 0; + entries_.swap(entries); } -void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl* tbl, - uint32_t max_bytes) { - if (tbl->max_bytes == max_bytes) { +void HPackTable::SetMaxBytes(uint32_t max_bytes) { + if (max_bytes_ == max_bytes) { return; } if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { gpr_log(GPR_INFO, "Update hpack parser max size to %d", max_bytes); } - while (tbl->mem_used > max_bytes) { - evict1(tbl); + while (mem_used_ > max_bytes) { + EvictOne(); } - tbl->max_bytes = max_bytes; + max_bytes_ = max_bytes; } -grpc_error_handle grpc_chttp2_hptbl_set_current_table_size( - grpc_chttp2_hptbl* tbl, uint32_t bytes) { - if (tbl->current_table_bytes == bytes) { +grpc_error_handle HPackTable::SetCurrentTableSize(uint32_t bytes) { + if (current_table_bytes_ == bytes) { return GRPC_ERROR_NONE; } - if (bytes > tbl->max_bytes) { + if (bytes > max_bytes_) { return GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrFormat( "Attempt to make hpack table %d bytes when max is %d bytes", bytes, - tbl->max_bytes) + max_bytes_) .c_str()); } if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) { gpr_log(GPR_INFO, "Update hpack parser table size to %d", bytes); } - while (tbl->mem_used > bytes) { - evict1(tbl); - } - tbl->current_table_bytes = bytes; - tbl->max_entries = grpc_chttp2_hptbl::entries_for_bytes(bytes); - if (tbl->max_entries > tbl->cap_entries) { - rebuild_ents(tbl, GPR_MAX(tbl->max_entries, 2 * tbl->cap_entries)); - } else if (tbl->max_entries < tbl->cap_entries / 3) { - uint32_t new_cap = GPR_MAX(tbl->max_entries, 16u); - if (new_cap != tbl->cap_entries) { - rebuild_ents(tbl, new_cap); + while (mem_used_ > bytes) { + EvictOne(); + } + current_table_bytes_ = bytes; + max_entries_ = EntriesForBytes(bytes); + if (max_entries_ > entries_.size()) { + Rebuild(max_entries_); + } else if (max_entries_ < entries_.size() / 3) { + // TODO(ctiller): move to resource quota system, only shrink under memory + // pressure + uint32_t new_cap = std::max(max_entries_, kInlineEntries); + if (new_cap != entries_.size()) { + Rebuild(new_cap); } } return GRPC_ERROR_NONE; } -grpc_error_handle grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl, - grpc_mdelem md) { +grpc_error_handle HPackTable::Add(grpc_mdelem md) { /* determine how many bytes of buffer this entry represents */ size_t elem_bytes = GRPC_SLICE_LENGTH(GRPC_MDKEY(md)) + - GRPC_SLICE_LENGTH(GRPC_MDVALUE(md)) + - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; + GRPC_SLICE_LENGTH(GRPC_MDVALUE(md)) + kEntryOverhead; - if (tbl->current_table_bytes > tbl->max_bytes) { + if (current_table_bytes_ > max_bytes_) { return GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrFormat( "HPACK max table size reduced to %d but not reflected by hpack " "stream (still at %d)", - tbl->max_bytes, tbl->current_table_bytes) + max_bytes_, current_table_bytes_) .c_str()); } - /* we can't add elements bigger than the max table size */ - if (elem_bytes > tbl->current_table_bytes) { - /* HPACK draft 10 section 4.4 states: - * If the size of the new entry is less than or equal to the maximum - * size, that entry is added to the table. It is not an error to - * attempt to add an entry that is larger than the maximum size; an - * attempt to add an entry larger than the entire table causes - * the table - * to be emptied of all existing entries, and results in an - * empty table. - */ - while (tbl->num_ents) { - evict1(tbl); + // we can't add elements bigger than the max table size + if (elem_bytes > current_table_bytes_) { + // HPACK draft 10 section 4.4 states: + // If the size of the new entry is less than or equal to the maximum + // size, that entry is added to the table. It is not an error to + // attempt to add an entry that is larger than the maximum size; an + // attempt to add an entry larger than the entire table causes + // the table to be emptied of all existing entries, and results in an + // empty table. + while (num_entries_) { + EvictOne(); } return GRPC_ERROR_NONE; } - /* evict entries to ensure no overflow */ - while (elem_bytes > - static_cast(tbl->current_table_bytes) - tbl->mem_used) { - evict1(tbl); + // evict entries to ensure no overflow + while (elem_bytes > static_cast(current_table_bytes_) - mem_used_) { + EvictOne(); } - /* copy the finalized entry in */ - tbl->ents[(tbl->first_ent + tbl->num_ents) % tbl->cap_entries] = + // copy the finalized entry in + entries_[(first_entry_ + num_entries_) % entries_.size()] = GRPC_MDELEM_REF(md); - /* update accounting values */ - tbl->num_ents++; - tbl->mem_used += static_cast(elem_bytes); + // update accounting values + num_entries_++; + mem_used_ += static_cast(elem_bytes); return GRPC_ERROR_NONE; } -grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( - const grpc_chttp2_hptbl* tbl, grpc_mdelem md) { - grpc_chttp2_hptbl_find_result r = {0, 0}; - uint32_t i; - - /* See if the string is in the static table */ - for (i = 0; i < GRPC_CHTTP2_LAST_STATIC_ENTRY; i++) { - grpc_mdelem ent = grpc_static_mdelem_manifested()[i]; - if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; - r.index = i + 1u; - r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); - if (r.has_value) return r; - } - - /* Scan the dynamic table */ - for (i = 0; i < tbl->num_ents; i++) { - uint32_t idx = static_cast(tbl->num_ents - i + - GRPC_CHTTP2_LAST_STATIC_ENTRY); - grpc_mdelem ent = tbl->ents[(tbl->first_ent + i) % tbl->cap_entries]; - if (!grpc_slice_eq(GRPC_MDKEY(md), GRPC_MDKEY(ent))) continue; - r.index = idx; - r.has_value = grpc_slice_eq(GRPC_MDVALUE(md), GRPC_MDVALUE(ent)); - if (r.has_value) return r; - } - - return r; -} +} // namespace grpc_core static size_t get_base64_encoded_size(size_t raw_length) { static const uint8_t tail_xtra[3] = {0, 2, 3}; diff --git a/src/core/ext/transport/chttp2/transport/hpack_table.h b/src/core/ext/transport/chttp2/transport/hpack_table.h index 14a77932a2e..25129a9b11b 100644 --- a/src/core/ext/transport/chttp2/transport/hpack_table.h +++ b/src/core/ext/transport/chttp2/transport/hpack_table.h @@ -27,98 +27,111 @@ #include "src/core/lib/transport/metadata.h" #include "src/core/lib/transport/static_metadata.h" -/* HPACK header table */ - -/* last index in the static table */ -#define GRPC_CHTTP2_LAST_STATIC_ENTRY 61 - -/* Initial table size as per the spec */ -#define GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE 4096 -/* Maximum table size that we'll use */ -#define GRPC_CHTTP2_MAX_HPACK_TABLE_SIZE GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE -/* Per entry overhead bytes as per the spec */ -#define GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD 32 -#if 0 -/* Maximum number of entries we could possibly fit in the table, given defined - overheads */ -#define GRPC_CHTTP2_MAX_TABLE_COUNT \ - ((GRPC_CHTTP2_MAX_HPACK_TABLE_SIZE + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / \ - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD) -#endif - -/* hpack decoder table */ -struct grpc_chttp2_hptbl { - static uint32_t entries_for_bytes(uint32_t bytes) { - return (bytes + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - 1) / - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; - } - static constexpr uint32_t kInitialCapacity = - (GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE + GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD - - 1) / - GRPC_CHTTP2_HPACK_ENTRY_OVERHEAD; - - grpc_chttp2_hptbl() { - GPR_DEBUG_ASSERT(!ents); - constexpr uint32_t AllocSize = sizeof(*ents) * kInitialCapacity; - ents = static_cast(gpr_malloc(AllocSize)); - memset(ents, 0, AllocSize); - } +namespace grpc_core { - /* the first used entry in ents */ - uint32_t first_ent = 0; - /* how many entries are in the table */ - uint32_t num_ents = 0; - /* the amount of memory used by the table, according to the hpack algorithm */ - uint32_t mem_used = 0; - /* the max memory allowed to be used by the table, according to the hpack - algorithm */ - uint32_t max_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; - /* the currently agreed size of the table, according to the hpack algorithm */ - uint32_t current_table_bytes = GRPC_CHTTP2_INITIAL_HPACK_TABLE_SIZE; - /* Maximum number of entries we could possibly fit in the table, given defined - overheads */ - uint32_t max_entries = kInitialCapacity; - /* Number of entries allocated in ents */ - uint32_t cap_entries = kInitialCapacity; - /* a circular buffer of headers - this is stored in the opposite order to - what hpack specifies, in order to simplify table management a little... - meaning lookups need to SUBTRACT from the end position */ - grpc_mdelem* ents = nullptr; -}; +namespace hpack_table_detail { +// Per entry overhead bytes as per the spec +static constexpr uint32_t kEntryOverhead = 32; +// Initial table size as per the spec +static constexpr uint32_t kInitialTableSize = 4096; + +static constexpr uint32_t EntriesForBytes(uint32_t bytes) noexcept { + return (bytes + kEntryOverhead - 1) / kEntryOverhead; +} + +static constexpr uint32_t kInlineEntries = EntriesForBytes(kInitialTableSize); +} // namespace hpack_table_detail + +// HPACK header table +class HPackTable { + public: + // last index in the static table + static constexpr uint32_t kLastStaticEntry = 61; + // Initial table size as per the spec + static constexpr uint32_t kInitialTableSize = + hpack_table_detail::kInitialTableSize; + // Per entry overhead bytes as per the spec + static constexpr uint32_t kEntryOverhead = hpack_table_detail::kEntryOverhead; + + HPackTable(); + ~HPackTable(); + + HPackTable(const HPackTable&); + HPackTable& operator=(const HPackTable&); + + void SetMaxBytes(uint32_t max_bytes); + grpc_error_handle SetCurrentTableSize(uint32_t bytes); + + // Lookup, but don't ref. + grpc_mdelem Peek(uint32_t index) const { return Lookup(index); } + // Lookup, taking a ref if found. + grpc_mdelem Fetch(uint32_t index) const { return Lookup(index); } + + // add a table entry to the index + grpc_error_handle Add(grpc_mdelem md) GRPC_MUST_USE_RESULT; -void grpc_chttp2_hptbl_destroy(grpc_chttp2_hptbl* tbl); -void grpc_chttp2_hptbl_set_max_bytes(grpc_chttp2_hptbl* tbl, - uint32_t max_bytes); -grpc_error_handle grpc_chttp2_hptbl_set_current_table_size( - grpc_chttp2_hptbl* tbl, uint32_t bytes); - -/* lookup a table entry based on its hpack index */ -grpc_mdelem grpc_chttp2_hptbl_lookup_dynamic_index(const grpc_chttp2_hptbl* tbl, - uint32_t tbl_index); -grpc_mdelem grpc_chttp2_hptbl_lookup_ref_dynamic_index( - const grpc_chttp2_hptbl* tbl, uint32_t tbl_index); -template -inline grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl, - uint32_t index) { - /* Static table comes first, just return an entry from it. - NB: This imposes the constraint that the first - GRPC_CHTTP2_LAST_STATIC_ENTRY entries in the core static metadata table - must follow the hpack standard. If that changes, we *must* not rely on - reading the core static metadata table here; at that point we'd need our - own singleton static metadata in the correct order. */ - if (index <= GRPC_CHTTP2_LAST_STATIC_ENTRY) { - return grpc_static_mdelem_manifested()[index - 1]; - } else { - if (take_ref) { - return grpc_chttp2_hptbl_lookup_ref_dynamic_index(tbl, index); + // Current entry count in the table. + uint32_t num_entries() const { return num_entries_; } + + private: + using EntriesVec = + absl::InlinedVector; + + /* lookup a table entry based on its hpack index */ + template + grpc_mdelem Lookup(uint32_t index) const { + // Static table comes first, just return an entry from it. + // NB: This imposes the constraint that the first + // GRPC_CHTTP2_LAST_STATIC_ENTRY entries in the core static metadata table + // must follow the hpack standard. If that changes, we *must* not rely on + // reading the core static metadata table here; at that point we'd need our + // own singleton static metadata in the correct order. + if (index <= kLastStaticEntry) { + return grpc_static_mdelem_manifested()[index - 1]; } else { - return grpc_chttp2_hptbl_lookup_dynamic_index(tbl, index); + return LookupDynamic(index); } } -} -/* add a table entry to the index */ -grpc_error_handle grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl, - grpc_mdelem md) GRPC_MUST_USE_RESULT; + + template + grpc_mdelem LookupDynamic(uint32_t index) const { + // Not static - find the value in the list of valid entries + const uint32_t tbl_index = index - (kLastStaticEntry + 1); + if (tbl_index < num_entries_) { + uint32_t offset = + (num_entries_ - 1u - tbl_index + first_entry_) % entries_.size(); + grpc_mdelem md = entries_[offset]; + if (take_ref) { + GRPC_MDELEM_REF(md); + } + return md; + } + // Invalid entry: return error + return GRPC_MDNULL; + } + + void EvictOne(); + void Rebuild(uint32_t new_cap); + + // The first used entry in ents. + uint32_t first_entry_ = 0; + // How many entries are in the table. + uint32_t num_entries_ = 0; + // The amount of memory used by the table, according to the hpack algorithm + uint32_t mem_used_ = 0; + // The max memory allowed to be used by the table, according to the hpack + // algorithm. + uint32_t max_bytes_ = kInitialTableSize; + // The currently agreed size of the table, according to the hpack algorithm. + uint32_t current_table_bytes_ = kInitialTableSize; + // Maximum number of entries we could possibly fit in the table, given defined + // overheads. + uint32_t max_entries_ = hpack_table_detail::kInlineEntries; + // HPack table entries + EntriesVec entries_; +}; + +} // namespace grpc_core size_t grpc_chttp2_get_size_in_hpack_table(grpc_mdelem elem, bool use_true_binary_metadata); @@ -130,19 +143,10 @@ inline uintptr_t grpc_chttp2_get_static_hpack_table_index(grpc_mdelem md) { uintptr_t index = reinterpret_cast(GRPC_MDELEM_DATA(md)) - grpc_static_mdelem_table(); - if (index < GRPC_CHTTP2_LAST_STATIC_ENTRY) { + if (index < grpc_core::HPackTable::kLastStaticEntry) { return index + 1; // Hpack static metadata element indices start at 1 } return 0; } -/* Find a key/value pair in the table... returns the index in the table of the - most similar entry, or 0 if the value was not found */ -struct grpc_chttp2_hptbl_find_result { - uint32_t index; - int has_value; -}; -grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find( - const grpc_chttp2_hptbl* tbl, grpc_mdelem md); - #endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_TABLE_H */ diff --git a/src/core/ext/transport/chttp2/transport/parsing.cc b/src/core/ext/transport/chttp2/transport/parsing.cc index 5d913358105..2b9fb6bba95 100644 --- a/src/core/ext/transport/chttp2/transport/parsing.cc +++ b/src/core/ext/transport/chttp2/transport/parsing.cc @@ -767,8 +767,7 @@ static grpc_error_handle init_settings_frame_parser(grpc_chttp2_transport* t) { if (t->incoming_frame_flags & GRPC_CHTTP2_FLAG_ACK) { memcpy(t->settings[GRPC_ACKED_SETTINGS], t->settings[GRPC_SENT_SETTINGS], GRPC_CHTTP2_NUM_SETTINGS * sizeof(uint32_t)); - grpc_chttp2_hptbl_set_max_bytes( - t->hpack_parser.hpack_table(), + t->hpack_parser.hpack_table()->SetMaxBytes( t->settings[GRPC_ACKED_SETTINGS] [GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]); t->sent_local_settings = false; diff --git a/test/core/transport/chttp2/binary-metadata.headers b/test/core/transport/chttp2/binary-metadata.headers new file mode 100644 index 00000000000..2fa9eef908d --- /dev/null +++ b/test/core/transport/chttp2/binary-metadata.headers @@ -0,0 +1,2 @@ +a.b.c-bin: b21nMjAyMQ== + diff --git a/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc b/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc index 7be46ab6cb4..3914fc7d92b 100644 --- a/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc +++ b/test/core/transport/chttp2/hpack_parser_fuzzer_test.cc @@ -45,7 +45,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { grpc_core::HPackParser parser; parser.BeginFrame(onhdr, grpc_core::HPackParser::Boundary::None, grpc_core::HPackParser::Priority::None); - GRPC_ERROR_UNREF(parser.Parse(grpc_slice_from_static_buffer(data, size))); + GRPC_ERROR_UNREF( + parser.Parse(grpc_slice_from_static_buffer(data, size), true)); } grpc_shutdown(); return 0; diff --git a/test/core/transport/chttp2/hpack_parser_test.cc b/test/core/transport/chttp2/hpack_parser_test.cc index 1328366f1b8..6b052f9fb6f 100644 --- a/test/core/transport/chttp2/hpack_parser_test.cc +++ b/test/core/transport/chttp2/hpack_parser_test.cc @@ -65,7 +65,11 @@ static void test_vector(grpc_core::HPackParser* parser, for (i = 0; i < nslices; i++) { grpc_core::ExecCtx exec_ctx; - GPR_ASSERT(parser->Parse(slices[i]) == GRPC_ERROR_NONE); + auto err = parser->Parse(slices[i], i == nslices - 1); + if (err != GRPC_ERROR_NONE) { + gpr_log(GPR_ERROR, "Unexpected parse error: %s", grpc_error_string(err)); + abort(); + } } for (i = 0; i < nslices; i++) { @@ -158,8 +162,8 @@ static void test_vectors(grpc_slice_split_mode mode) { { grpc_core::HPackParser parser; - grpc_chttp2_hptbl_set_max_bytes(parser.hpack_table(), 256); - grpc_chttp2_hptbl_set_current_table_size(parser.hpack_table(), 256); + parser.hpack_table()->SetMaxBytes(256); + parser.hpack_table()->SetCurrentTableSize(256); /* D.5.1 */ test_vector(&parser, mode, "4803 3330 3258 0770 7269 7661 7465 611d" @@ -199,8 +203,8 @@ static void test_vectors(grpc_slice_split_mode mode) { { grpc_core::HPackParser parser; - grpc_chttp2_hptbl_set_max_bytes(parser.hpack_table(), 256); - grpc_chttp2_hptbl_set_current_table_size(parser.hpack_table(), 256); + parser.hpack_table()->SetMaxBytes(256); + parser.hpack_table()->SetCurrentTableSize(256); /* D.6.1 */ test_vector(&parser, mode, "4882 6402 5885 aec3 771a 4b61 96d0 7abe" @@ -234,6 +238,18 @@ static void test_vectors(grpc_slice_split_mode mode) { "set-cookie", "foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1")}); } + + { + grpc_core::HPackParser parser; + // Binary metadata: created using: + // tools/codegen/core/gen_header_frame.py + // --compression inc --no_framing --hex + // < test/core/transport/chttp2/binary-metadata.headers + test_vector(&parser, mode, + "40 09 61 2e 62 2e 63 2d 62 69 6e 0c 62 32 31 6e 4d 6a 41 79 " + "4d 51 3d 3d", + {std::make_pair("a.b.c-bin", "omg2021")}); + } } int main(int argc, char** argv) { diff --git a/test/core/transport/chttp2/hpack_table_test.cc b/test/core/transport/chttp2/hpack_table_test.cc index c8410c61866..04b69a9b633 100644 --- a/test/core/transport/chttp2/hpack_table_test.cc +++ b/test/core/transport/chttp2/hpack_table_test.cc @@ -36,21 +36,23 @@ #define LOG_TEST(x) gpr_log(GPR_INFO, "%s", x) -static void assert_str(const grpc_chttp2_hptbl* /*tbl*/, grpc_slice mdstr, +using grpc_core::HPackTable; + +static void assert_str(const HPackTable* /*tbl*/, grpc_slice mdstr, const char* str) { GPR_ASSERT(grpc_slice_str_cmp(mdstr, str) == 0); } -static void assert_index(const grpc_chttp2_hptbl* tbl, uint32_t idx, - const char* key, const char* value) { - grpc_mdelem md = grpc_chttp2_hptbl_lookup(tbl, idx); +static void assert_index(const HPackTable* tbl, uint32_t idx, const char* key, + const char* value) { + grpc_mdelem md = tbl->Peek(idx); assert_str(tbl, GRPC_MDKEY(md), key); assert_str(tbl, GRPC_MDVALUE(md), value); } static void test_static_lookup(void) { grpc_core::ExecCtx exec_ctx; - grpc_chttp2_hptbl tbl; + HPackTable tbl; LOG_TEST("test_static_lookup"); assert_index(&tbl, 1, ":authority", ""); @@ -114,12 +116,10 @@ static void test_static_lookup(void) { assert_index(&tbl, 59, "vary", ""); assert_index(&tbl, 60, "via", ""); assert_index(&tbl, 61, "www-authenticate", ""); - - grpc_chttp2_hptbl_destroy(&tbl); } static void test_many_additions(void) { - grpc_chttp2_hptbl tbl; + HPackTable tbl; int i; LOG_TEST("test_many_additions"); @@ -132,135 +132,17 @@ static void test_many_additions(void) { std::string value = absl::StrCat("VALUE:", i); elem = grpc_mdelem_from_slices(grpc_slice_from_cpp_string(key), grpc_slice_from_cpp_string(value)); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); + GPR_ASSERT(tbl.Add(elem) == GRPC_ERROR_NONE); GRPC_MDELEM_UNREF(elem); - assert_index(&tbl, 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY, key.c_str(), + assert_index(&tbl, 1 + HPackTable::kLastStaticEntry, key.c_str(), value.c_str()); if (i) { std::string key = absl::StrCat("K:", i - 1); std::string value = absl::StrCat("VALUE:", i - 1); - assert_index(&tbl, 2 + GRPC_CHTTP2_LAST_STATIC_ENTRY, key.c_str(), + assert_index(&tbl, 2 + HPackTable::kLastStaticEntry, key.c_str(), value.c_str()); } } - - grpc_chttp2_hptbl_destroy(&tbl); -} - -static grpc_chttp2_hptbl_find_result find_simple(grpc_chttp2_hptbl* tbl, - const char* key, - const char* value) { - grpc_core::ExecCtx exec_ctx; - grpc_mdelem md = grpc_mdelem_from_slices( - grpc_slice_from_copied_string(key), grpc_slice_from_copied_string(value)); - grpc_chttp2_hptbl_find_result r = grpc_chttp2_hptbl_find(tbl, md); - GRPC_MDELEM_UNREF(md); - - return r; -} - -static void test_find(void) { - grpc_core::ExecCtx exec_ctx; - grpc_chttp2_hptbl tbl; - uint32_t i; - char buffer[32]; - grpc_mdelem elem; - grpc_chttp2_hptbl_find_result r; - - LOG_TEST("test_find"); - - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("abc"), - grpc_slice_from_static_string("xyz")); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("abc"), - grpc_slice_from_static_string("123")); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("x"), - grpc_slice_from_static_string("1")); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - - r = find_simple(&tbl, "abc", "123"); - GPR_ASSERT(r.index == 2 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "abc", "xyz"); - GPR_ASSERT(r.index == 3 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "x", "1"); - GPR_ASSERT(r.index == 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "x", "2"); - GPR_ASSERT(r.index == 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "vary", "some-vary-arg"); - GPR_ASSERT(r.index == 59); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "accept-encoding", "gzip, deflate"); - GPR_ASSERT(r.index == 16); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "accept-encoding", "gzip"); - GPR_ASSERT(r.index == 16); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, ":method", "GET"); - GPR_ASSERT(r.index == 2); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, ":method", "POST"); - GPR_ASSERT(r.index == 3); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, ":method", "PUT"); - GPR_ASSERT(r.index == 2 || r.index == 3); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "this-does-not-exist", ""); - GPR_ASSERT(r.index == 0); - GPR_ASSERT(r.has_value == 0); - - /* overflow the string buffer, check find still works */ - for (i = 0; i < 10000; i++) { - int64_ttoa(i, buffer); - elem = grpc_mdelem_from_slices(grpc_slice_from_static_string("test"), - grpc_slice_from_copied_string(buffer)); - GPR_ASSERT(grpc_chttp2_hptbl_add(&tbl, elem) == GRPC_ERROR_NONE); - GRPC_MDELEM_UNREF(elem); - } - - r = find_simple(&tbl, "abc", "123"); - GPR_ASSERT(r.index == 0); - GPR_ASSERT(r.has_value == 0); - - r = find_simple(&tbl, "test", "9999"); - GPR_ASSERT(r.index == 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - r = find_simple(&tbl, "test", "9998"); - GPR_ASSERT(r.index == 2 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - - for (i = 0; i < tbl.num_ents; i++) { - uint32_t expect = 9999 - i; - int64_ttoa(expect, buffer); - - r = find_simple(&tbl, "test", buffer); - GPR_ASSERT(r.index == i + 1 + GRPC_CHTTP2_LAST_STATIC_ENTRY); - GPR_ASSERT(r.has_value == 1); - } - - r = find_simple(&tbl, "test", "10000"); - GPR_ASSERT(r.index != 0); - GPR_ASSERT(r.has_value == 0); - - grpc_chttp2_hptbl_destroy(&tbl); } int main(int argc, char** argv) { @@ -268,7 +150,6 @@ int main(int argc, char** argv) { grpc_init(); test_static_lookup(); test_many_additions(); - test_find(); grpc_shutdown(); return 0; } diff --git a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc index 5a29df607da..a7e645666ae 100644 --- a/test/cpp/microbenchmarks/bm_chttp2_hpack.cc +++ b/test/cpp/microbenchmarks/bm_chttp2_hpack.cc @@ -461,11 +461,11 @@ static void BM_HpackParserParseHeader(benchmark::State& state) { grpc_core::HPackParser::Boundary::None, grpc_core::HPackParser::Priority::None); for (auto slice : init_slices) { - GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice)); + GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice, false)); } while (state.KeepRunning()) { for (auto slice : benchmark_slices) { - GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice)); + GPR_ASSERT(GRPC_ERROR_NONE == p.Parse(slice, false)); } grpc_core::ExecCtx::Get()->Flush(); // Recreate arena every 4k iterations to avoid oom diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 275b10c3eb0..90df4102a4c 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -1775,10 +1775,12 @@ src/core/lib/gprpp/global_config_generic.h \ src/core/lib/gprpp/host_port.cc \ src/core/lib/gprpp/host_port.h \ src/core/lib/gprpp/manual_constructor.h \ +src/core/lib/gprpp/match.h \ src/core/lib/gprpp/memory.h \ src/core/lib/gprpp/mpscq.cc \ src/core/lib/gprpp/mpscq.h \ src/core/lib/gprpp/orphanable.h \ +src/core/lib/gprpp/overload.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ src/core/lib/gprpp/stat.h \ diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index 1e16ac3aa03..0fd55c17c47 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -1614,10 +1614,12 @@ src/core/lib/gprpp/global_config_generic.h \ src/core/lib/gprpp/host_port.cc \ src/core/lib/gprpp/host_port.h \ src/core/lib/gprpp/manual_constructor.h \ +src/core/lib/gprpp/match.h \ src/core/lib/gprpp/memory.h \ src/core/lib/gprpp/mpscq.cc \ src/core/lib/gprpp/mpscq.h \ src/core/lib/gprpp/orphanable.h \ +src/core/lib/gprpp/overload.h \ src/core/lib/gprpp/ref_counted.h \ src/core/lib/gprpp/ref_counted_ptr.h \ src/core/lib/gprpp/stat.h \