Chiebot-Mirror
/
grpc
mirror of https://github.com/grpc/grpc.git
8
0
Fork 0
Code Issues Projects Releases Wiki Activity

Convert HPACK encoder to C++ (#27226)

Browse Source 
* Rebuild HPACK encoder table as C++

* move comment

* incguards

* build

* Automated change: Fix sanity tests

* c++ initialization ftw

* Automated change: Fix sanity tests

* Add missing header

* Add missing header

* Begin converting HPACK encoder to c++

* First pass conversion to c++

* fixes

* Automated change: Fix sanity tests

Co-authored-by: ctiller <ctiller@users.noreply.github.com>
pull/27271/head
Craig Tiller 3 years ago committed by GitHub
parent 483e11ddc3
commit dc701787e2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 559 additions and 554 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 13
      src/core/ext/transport/chttp2/transport/chttp2_transport.cc
  2. 656
      src/core/ext/transport/chttp2/transport/hpack_encoder.cc
  3. 208
      src/core/ext/transport/chttp2/transport/hpack_encoder.h
  4. 2
      src/core/ext/transport/chttp2/transport/internal.h
  5. 10
      src/core/ext/transport/chttp2/transport/varint.cc
  6. 67
      src/core/ext/transport/chttp2/transport/varint.h
  7. 53
      src/core/ext/transport/chttp2/transport/writing.cc
  8. 8
      src/core/lib/transport/metadata_batch.h
  9. 33
      test/core/transport/chttp2/hpack_encoder_test.cc
  10. 14
      test/core/transport/chttp2/varint_test.cc
  11. 49
      test/cpp/microbenchmarks/bm_chttp2_hpack.cc

13
src/core/ext/transport/chttp2/transport/chttp2_transport.cc
Unescape View File

@ -204,7 +204,6 @@ grpc_chttp2_transport::~grpc_chttp2_transport() {
grpc_slice_buffer_destroy_internal(&qbuf);
grpc_slice_buffer_destroy_internal(&outbuf);
grpc_chttp2_hpack_compressor_destroy(&hpack_compressor);
grpc_error_handle error =
GRPC_ERROR_CREATE_FROM_STATIC_STRING("Transport destroyed");
@ -280,8 +279,7 @@ static bool read_channel_args(grpc_chttp2_transport* t,
const int value =
grpc_channel_arg_get_integer(&channel_args->args[i], options);
if (value >= 0) {
grpc_chttp2_hpack_compressor_set_max_usable_size(
&t->hpack_compressor, static_cast<uint32_t>(value));
t->hpack_compressor.SetMaxUsableSize(value);
}
} else if (0 == strcmp(channel_args->args[i].key,
GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA)) {
@ -474,7 +472,6 @@ grpc_chttp2_transport::grpc_chttp2_transport(
grpc_slice_buffer_add(&outbuf, grpc_slice_from_copied_string(
GRPC_CHTTP2_CLIENT_CONNECT_STRING));
}
grpc_chttp2_hpack_compressor_init(&hpack_compressor);
grpc_slice_buffer_init(&qbuf);
// copy in initial settings to all setting sets
size_t i;
@ -2352,8 +2349,8 @@ static void close_from_api(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
size_t msg_len = GRPC_SLICE_LENGTH(slice);
GPR_ASSERT(msg_len <= UINT32_MAX);
uint32_t msg_len_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)msg_len, 1);
message_pfx = GRPC_SLICE_MALLOC(14 + msg_len_len);
grpc_core::VarintWriter<1> msg_len_writer(msg_len);
message_pfx = GRPC_SLICE_MALLOC(14 + msg_len_writer.length());
p = GRPC_SLICE_START_PTR(message_pfx);
*p++ = 0x00; /* literal header, not indexed */
*p++ = 12; /* len(grpc-message) */
@ -2369,8 +2366,8 @@ static void close_from_api(grpc_chttp2_transport* t, grpc_chttp2_stream* s,
*p++ = 'a';
*p++ = 'g';
*p++ = 'e';
GRPC_CHTTP2_WRITE_VARINT((uint32_t)msg_len, 1, 0, p, (uint32_t)msg_len_len);
p += msg_len_len;
msg_len_writer.Write(0, p);
p += msg_len_writer.length();
GPR_ASSERT(p == GRPC_SLICE_END_PTR(message_pfx));
len += static_cast<uint32_t> GRPC_SLICE_LENGTH(message_pfx);
len += static_cast<uint32_t>(msg_len);

656
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
Unescape View File

@ -42,28 +42,9 @@
#include "src/core/lib/transport/static_metadata.h"
#include "src/core/lib/transport/timeout_encoding.h"
namespace grpc_core {
namespace {
/* (Maybe-cuckoo) hpack encoder hash table implementation.
This hashtable implementation is a subset of a proper cuckoo hash; while we
have fallback cells that a value can be hashed to if the first cell is full,
we do not attempt to iteratively rearrange entries into backup cells to get
things to fit. Instead, if both a cell and the backup cell for a value are
occupied, the older existing entry is evicted.
Note that we can disable backup-cell picking by setting
GRPC_HPACK_ENCODER_USE_CUCKOO_HASH to 0. In that case, we simply evict an
existing entry rather than try to use a backup. Hence, "maybe-cuckoo."
TODO(arjunroy): Add unit tests for hashtable implementation. */
#define GRPC_HPACK_ENCODER_USE_CUCKOO_HASH 1
#define HASH_FRAGMENT_MASK (GRPC_CHTTP2_HPACKC_NUM_VALUES - 1)
#define HASH_FRAGMENT_1(x) ((x)&HASH_FRAGMENT_MASK)
#define HASH_FRAGMENT_2(x) \
(((x) >> GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS) & HASH_FRAGMENT_MASK)
#define HASH_FRAGMENT_3(x) \
(((x) >> (GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS * 2)) & HASH_FRAGMENT_MASK)
#define HASH_FRAGMENT_4(x) \
(((x) >> (GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS * 3)) & HASH_FRAGMENT_MASK)
/* don't consider adding anything bigger than this to the hpack table */
constexpr size_t kMaxDecoderSpaceUsage = 512;
@ -71,40 +52,20 @@ constexpr size_t kDataFrameHeaderSize = 9;
} /* namespace */
struct framer_state {
int is_first_frame;
/* number of bytes in 'output' when we started the frame - used to calculate
frame length */
size_t output_length_at_start_of_frame;
/* index (in output) of the header for the current frame */
size_t header_idx;
#ifndef NDEBUG
/* have we seen a regular (non-colon-prefixed) header yet? */
uint8_t seen_regular_header;
#endif
/* output stream id */
uint32_t stream_id;
grpc_slice_buffer* output;
grpc_transport_one_way_stats* stats;
/* maximum size of a frame */
size_t max_frame_size;
bool use_true_binary_metadata;
bool is_end_of_stream;
};
/* fills p (which is expected to be kDataFrameHeaderSize bytes long)
* with a data frame header */
static void fill_header(uint8_t* p, uint8_t type, uint32_t id, size_t len,
uint8_t flags) {
static void FillHeader(uint8_t* p, uint8_t type, uint32_t id, size_t len,
uint8_t flags) {
/* len is the current frame size (i.e. for the frame we're finishing).
We finish a frame if:
1) We called ensure_space(), (i.e. add_tiny_header_data()) and adding
'need_bytes' to the frame would cause us to exceed st->max_frame_size.
'need_bytes' to the frame would cause us to exceed max_frame_size.
2) We called add_header_data, and adding the slice would cause us to exceed
st->max_frame_size.
max_frame_size.
3) We're done encoding the header.
Thus, len is always <= st->max_frame_size.
st->max_frame_size is derived from GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE,
Thus, len is always <= max_frame_size.
max_frame_size is derived from GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE,
which has a max allowable value of 16777215 (see chttp_transport.cc).
Thus, the following assert can be a debug assert. */
GPR_DEBUG_ASSERT(len < 16777316);
@ -119,131 +80,123 @@ static void fill_header(uint8_t* p, uint8_t type, uint32_t id, size_t len,
*p++ = static_cast<uint8_t>(id);
}
static size_t current_frame_size(framer_state* st) {
size_t HPackCompressor::Framer::CurrentFrameSize() const {
const size_t frame_size =
st->output->length - st->output_length_at_start_of_frame;
GPR_DEBUG_ASSERT(frame_size <= st->max_frame_size);
output_->length - prefix_.output_length_at_start_of_frame;
GPR_DEBUG_ASSERT(frame_size <= max_frame_size_);
return frame_size;
}
/* finish a frame - fill in the previously reserved header */
static void finish_frame(framer_state* st, int is_header_boundary) {
uint8_t type = 0xff;
type =
static_cast<uint8_t>(st->is_first_frame ? GRPC_CHTTP2_FRAME_HEADER
: GRPC_CHTTP2_FRAME_CONTINUATION);
uint8_t flags = 0xff;
/* per the HTTP/2 spec:
A HEADERS frame carries the END_STREAM flag that signals the end of a
stream. However, a HEADERS frame with the END_STREAM flag set can be
followed by CONTINUATION frames on the same stream. Logically, the
CONTINUATION frames are part of the HEADERS frame.
Thus, we add the END_STREAM flag to the HEADER frame (the first frame). */
flags = static_cast<uint8_t>(st->is_first_frame && st->is_end_of_stream
? GRPC_CHTTP2_DATA_FLAG_END_STREAM
: 0);
/* per the HTTP/2 spec:
A HEADERS frame without the END_HEADERS flag set MUST be followed by
a CONTINUATION frame for the same stream.
Thus, we add the END_HEADER flag to the last frame. */
flags |= static_cast<uint8_t>(
is_header_boundary ? GRPC_CHTTP2_DATA_FLAG_END_HEADERS : 0);
fill_header(GRPC_SLICE_START_PTR(st->output->slices[st->header_idx]), type,
st->stream_id, current_frame_size(st), flags);
st->stats->framing_bytes += kDataFrameHeaderSize;
st->is_first_frame = 0;
// finish a frame - fill in the previously reserved header
void HPackCompressor::Framer::FinishFrame(bool is_header_boundary) {
const uint8_t type = is_first_frame_ ? GRPC_CHTTP2_FRAME_HEADER
: GRPC_CHTTP2_FRAME_CONTINUATION;
uint8_t flags = 0;
// per the HTTP/2 spec:
// A HEADERS frame carries the END_STREAM flag that signals the end of a
// stream. However, a HEADERS frame with the END_STREAM flag set can be
// followed by CONTINUATION frames on the same stream. Logically, the
// CONTINUATION frames are part of the HEADERS frame.
// Thus, we add the END_STREAM flag to the HEADER frame (the first frame).
if (is_first_frame_ && is_end_of_stream_) {
flags |= GRPC_CHTTP2_DATA_FLAG_END_STREAM;
}
// per the HTTP/2 spec:
// A HEADERS frame without the END_HEADERS flag set MUST be followed by
// a CONTINUATION frame for the same stream.
// Thus, we add the END_HEADER flag to the last frame.
if (is_header_boundary) {
flags |= GRPC_CHTTP2_DATA_FLAG_END_HEADERS;
}
FillHeader(GRPC_SLICE_START_PTR(output_->slices[prefix_.header_idx]), type,
stream_id_, CurrentFrameSize(), flags);
stats_->framing_bytes += kDataFrameHeaderSize;
is_first_frame_ = false;
}
/* begin a new frame: reserve off header space, remember how many bytes we'd
output before beginning */
static void begin_frame(framer_state* st) {
// begin a new frame: reserve off header space, remember how many bytes we'd
// output before beginning
HPackCompressor::Framer::FramePrefix HPackCompressor::Framer::BeginFrame() {
grpc_slice reserved;
reserved.refcount = nullptr;
reserved.data.inlined.length = kDataFrameHeaderSize;
st->header_idx = grpc_slice_buffer_add_indexed(st->output, reserved);
st->output_length_at_start_of_frame = st->output->length;
return FramePrefix{grpc_slice_buffer_add_indexed(output_, reserved),
output_->length};
}
/* make sure that the current frame is of the type desired, and has sufficient
space to add at least about_to_add bytes -- finishes the current frame if
needed */
static void ensure_space(framer_state* st, size_t need_bytes) {
if (GPR_LIKELY(current_frame_size(st) + need_bytes <= st->max_frame_size)) {
// make sure that the current frame is of the type desired, and has sufficient
// space to add at least about_to_add bytes -- finishes the current frame if
// needed
void HPackCompressor::Framer::EnsureSpace(size_t need_bytes) {
if (GPR_LIKELY(CurrentFrameSize() + need_bytes <= max_frame_size_)) {
return;
}
finish_frame(st, 0);
begin_frame(st);
FinishFrame(false);
prefix_ = BeginFrame();
}
static void add_header_data(framer_state* st, grpc_slice slice) {
size_t len = GRPC_SLICE_LENGTH(slice);
size_t remaining;
void HPackCompressor::Framer::Add(grpc_slice slice) {
const size_t len = GRPC_SLICE_LENGTH(slice);
if (len == 0) return;
remaining = st->max_frame_size - current_frame_size(st);
const size_t remaining = max_frame_size_ - CurrentFrameSize();
if (len <= remaining) {
st->stats->header_bytes += len;
grpc_slice_buffer_add(st->output, slice);
stats_->header_bytes += len;
grpc_slice_buffer_add(output_, slice);
} else {
st->stats->header_bytes += remaining;
grpc_slice_buffer_add(st->output, grpc_slice_split_head(&slice, remaining));
finish_frame(st, 0);
begin_frame(st);
add_header_data(st, slice);
stats_->header_bytes += remaining;
grpc_slice_buffer_add(output_, grpc_slice_split_head(&slice, remaining));
FinishFrame(false);
prefix_ = BeginFrame();
Add(slice);
}
}
static uint8_t* add_tiny_header_data(framer_state* st, size_t len) {
ensure_space(st, len);
st->stats->header_bytes += len;
return grpc_slice_buffer_tiny_add(st->output, len);
uint8_t* HPackCompressor::Framer::AddTiny(size_t len) {
EnsureSpace(len);
stats_->header_bytes += len;
return grpc_slice_buffer_tiny_add(output_, len);
}
// Add a key to the dynamic table. Both key and value will be added to table at
// the decoder.
static void AddKeyWithIndex(grpc_chttp2_hpack_compressor* c,
grpc_slice_refcount* key_ref, uint32_t new_index,
uint32_t key_hash) {
c->key_index->Insert(
grpc_chttp2_hpack_compressor::KeySliceRef(key_ref, key_hash), new_index);
void HPackCompressor::AddKeyWithIndex(grpc_slice_refcount* key_ref,
uint32_t new_index, uint32_t key_hash) {
key_index_.Insert(KeySliceRef(key_ref, key_hash), new_index);
}
/* add an element to the decoder table */
static void AddElemWithIndex(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
uint32_t new_index, uint32_t elem_hash,
uint32_t key_hash) {
void HPackCompressor::AddElemWithIndex(grpc_mdelem elem, uint32_t new_index,
uint32_t elem_hash, uint32_t key_hash) {
GPR_DEBUG_ASSERT(GRPC_MDELEM_IS_INTERNED(elem));
c->elem_index->Insert(grpc_chttp2_hpack_compressor::KeyElem(elem, elem_hash),
new_index);
AddKeyWithIndex(c, GRPC_MDKEY(elem).refcount, new_index, key_hash);
elem_index_.Insert(KeyElem(elem, elem_hash), new_index);
AddKeyWithIndex(GRPC_MDKEY(elem).refcount, new_index, key_hash);
}
static void add_elem(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
size_t elem_size, uint32_t elem_hash, uint32_t key_hash) {
uint32_t new_index = c->table->AllocateIndex(elem_size);
void HPackCompressor::AddElem(grpc_mdelem elem, size_t elem_size,
uint32_t elem_hash, uint32_t key_hash) {
uint32_t new_index = table_.AllocateIndex(elem_size);
if (new_index != 0) {
AddElemWithIndex(c, elem, new_index, elem_hash, key_hash);
AddElemWithIndex(elem, new_index, elem_hash, key_hash);
}
}
static void add_key(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
size_t elem_size, uint32_t key_hash) {
uint32_t new_index = c->table->AllocateIndex(elem_size);
void HPackCompressor::AddKey(grpc_mdelem elem, size_t elem_size,
uint32_t key_hash) {
uint32_t new_index = table_.AllocateIndex(elem_size);
if (new_index != 0) {
AddKeyWithIndex(c, GRPC_MDKEY(elem).refcount, new_index, key_hash);
AddKeyWithIndex(GRPC_MDKEY(elem).refcount, new_index, key_hash);
}
}
static void emit_indexed(grpc_chttp2_hpack_compressor* /*c*/,
uint32_t elem_index, framer_state* st) {
void HPackCompressor::Framer::EmitIndexed(uint32_t elem_index) {
GRPC_STATS_INC_HPACK_SEND_INDEXED();
uint32_t len = GRPC_CHTTP2_VARINT_LENGTH(elem_index, 1);
GRPC_CHTTP2_WRITE_VARINT(elem_index, 1, 0x80, add_tiny_header_data(st, len),
len);
VarintWriter<1> w(elem_index);
w.Write(0x80, AddTiny(w.length()));
}
struct wire_value {
wire_value(uint8_t huffman_prefix, bool insert_null_before_wire_value,
const grpc_slice& slice)
struct WireValue {
WireValue(uint8_t huffman_prefix, bool insert_null_before_wire_value,
const grpc_slice& slice)
: data(slice),
huffman_prefix(huffman_prefix),
insert_null_before_wire_value(insert_null_before_wire_value),
@ -257,124 +210,131 @@ struct wire_value {
const size_t length;
};
template <bool mdkey_definitely_interned>
static wire_value get_wire_value(grpc_mdelem elem, bool true_binary_enabled) {
const bool is_bin_hdr =
mdkey_definitely_interned
? grpc_is_refcounted_slice_binary_header(GRPC_MDKEY(elem))
: grpc_is_binary_header_internal(GRPC_MDKEY(elem));
const grpc_slice& value = GRPC_MDVALUE(elem);
static WireValue GetWireValue(const grpc_slice& value, bool true_binary_enabled,
bool is_bin_hdr) {
if (is_bin_hdr) {
if (true_binary_enabled) {
GRPC_STATS_INC_HPACK_SEND_BINARY();
return wire_value(0x00, true, grpc_slice_ref_internal(value));
return WireValue(0x00, true, grpc_slice_ref_internal(value));
} else {
GRPC_STATS_INC_HPACK_SEND_BINARY_BASE64();
return wire_value(0x80, false,
grpc_chttp2_base64_encode_and_huffman_compress(value));
return WireValue(0x80, false,
grpc_chttp2_base64_encode_and_huffman_compress(value));
}
} else {
/* TODO(ctiller): opportunistically compress non-binary headers */
GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED();
return wire_value(0x00, false, grpc_slice_ref_internal(value));
return WireValue(0x00, false, grpc_slice_ref_internal(value));
}
}
static uint32_t wire_value_length(const wire_value& v) {
GPR_DEBUG_ASSERT(v.length <= UINT32_MAX);
return static_cast<uint32_t>(v.length);
}
struct DefinitelyInterned {
static bool IsBinary(grpc_slice key) {
return grpc_is_refcounted_slice_binary_header(key);
}
};
struct UnsureIfInterned {
static bool IsBinary(grpc_slice key) {
return grpc_is_binary_header_internal(key);
}
};
namespace {
enum class EmitLitHdrType { INC_IDX, NO_IDX };
enum class EmitLitHdrVType { INC_IDX_V, NO_IDX_V };
} // namespace
template <EmitLitHdrType type>
static void emit_lithdr(grpc_chttp2_hpack_compressor* /*c*/, uint32_t key_index,
grpc_mdelem elem, framer_state* st) {
switch (type) {
case EmitLitHdrType::INC_IDX:
GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX();
break;
case EmitLitHdrType::NO_IDX:
GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX();
break;
class StringValue {
public:
template <typename MetadataKeyType>
StringValue(MetadataKeyType, grpc_mdelem elem, bool use_true_binary_metadata)
: wire_value_(GetWireValue(GRPC_MDVALUE(elem), use_true_binary_metadata,
MetadataKeyType::IsBinary(GRPC_MDKEY(elem)))),
len_val_(wire_value_.length) {}
size_t prefix_length() const {
return len_val_.length() +
(wire_value_.insert_null_before_wire_value ? 1 : 0);
}
const uint32_t len_pfx = type == EmitLitHdrType::INC_IDX
? GRPC_CHTTP2_VARINT_LENGTH(key_index, 2)
: GRPC_CHTTP2_VARINT_LENGTH(key_index, 4);
const wire_value value =
get_wire_value<true>(elem, st->use_true_binary_metadata);
const uint32_t len_val = wire_value_length(value);
const uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1);
GPR_DEBUG_ASSERT(len_pfx + len_val_len < GRPC_SLICE_INLINED_SIZE);
uint8_t* data = add_tiny_header_data(
st,
len_pfx + len_val_len + (value.insert_null_before_wire_value ? 1 : 0));
switch (type) {
case EmitLitHdrType::INC_IDX:
GRPC_CHTTP2_WRITE_VARINT(key_index, 2, 0x40, data, len_pfx);
break;
case EmitLitHdrType::NO_IDX:
GRPC_CHTTP2_WRITE_VARINT(key_index, 4, 0x00, data, len_pfx);
break;
void WritePrefix(uint8_t* prefix_data) {
len_val_.Write(wire_value_.huffman_prefix, prefix_data);
if (wire_value_.insert_null_before_wire_value) {
prefix_data[len_val_.length()] = 0;
}
}
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, value.huffman_prefix, &data[len_pfx],
len_val_len);
if (value.insert_null_before_wire_value) {
data[len_pfx + len_val_len] = 0;
const grpc_slice& data() { return wire_value_.data; }
private:
WireValue wire_value_;
VarintWriter<1> len_val_;
};
class StringKey {
public:
explicit StringKey(grpc_slice key)
: key_(key), len_key_(GRPC_SLICE_LENGTH(key)) {}
size_t prefix_length() const { return 1 + len_key_.length(); }
void WritePrefix(uint8_t type, uint8_t* data) {
data[0] = type;
len_key_.Write(0x00, data + 1);
}
add_header_data(st, value.data);
grpc_slice key() const { return key_; }
private:
grpc_slice key_;
VarintWriter<1> len_key_;
};
void HPackCompressor::Framer::EmitLitHdrIncIdx(uint32_t key_index,
grpc_mdelem elem) {
GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX();
StringValue emit(DefinitelyInterned(), elem, use_true_binary_metadata_);
VarintWriter<2> key(key_index);
uint8_t* data = AddTiny(key.length() + emit.prefix_length());
key.Write(0x40, data);
emit.WritePrefix(data + key.length());
Add(emit.data());
}
template <EmitLitHdrVType type>
static void emit_lithdr_v(grpc_chttp2_hpack_compressor* /*c*/, grpc_mdelem elem,
framer_state* st) {
switch (type) {
case EmitLitHdrVType::INC_IDX_V:
GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V();
break;
case EmitLitHdrVType::NO_IDX_V:
GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V();
break;
}
void HPackCompressor::Framer::EmitLitHdrNotIdx(uint32_t key_index,
grpc_mdelem elem) {
GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX();
StringValue emit(DefinitelyInterned(), elem, use_true_binary_metadata_);
VarintWriter<4> key(key_index);
uint8_t* data = AddTiny(key.length() + emit.prefix_length());
key.Write(0x00, data);
emit.WritePrefix(data + key.length());
Add(emit.data());
}
void HPackCompressor::Framer::EmitLitHdrWithStringKeyIncIdx(grpc_mdelem elem) {
GRPC_STATS_INC_HPACK_SEND_LITHDR_INCIDX_V();
GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED();
const uint32_t len_key =
static_cast<uint32_t>(GRPC_SLICE_LENGTH(GRPC_MDKEY(elem)));
const wire_value value =
type == EmitLitHdrVType::INC_IDX_V
? get_wire_value<true>(elem, st->use_true_binary_metadata)
: get_wire_value<false>(elem, st->use_true_binary_metadata);
const uint32_t len_val = wire_value_length(value);
const uint32_t len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1);
const uint32_t len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1);
GPR_DEBUG_ASSERT(len_key <= UINT32_MAX);
GPR_DEBUG_ASSERT(1 + len_key_len < GRPC_SLICE_INLINED_SIZE);
uint8_t* key_buf = add_tiny_header_data(st, 1 + len_key_len);
key_buf[0] = type == EmitLitHdrVType::INC_IDX_V ? 0x40 : 0x00;
GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00, &key_buf[1], len_key_len);
add_header_data(st, grpc_slice_ref_internal(GRPC_MDKEY(elem)));
uint8_t* value_buf = add_tiny_header_data(
st, len_val_len + (value.insert_null_before_wire_value ? 1 : 0));
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, value.huffman_prefix, value_buf,
len_val_len);
if (value.insert_null_before_wire_value) {
value_buf[len_val_len] = 0;
}
add_header_data(st, value.data);
StringKey key(GRPC_MDKEY(elem));
key.WritePrefix(0x40, AddTiny(key.prefix_length()));
Add(grpc_slice_ref_internal(key.key()));
StringValue emit(DefinitelyInterned(), elem, use_true_binary_metadata_);
emit.WritePrefix(AddTiny(emit.prefix_length()));
Add(emit.data());
}
static void emit_advertise_table_size_change(grpc_chttp2_hpack_compressor* c,
framer_state* st) {
uint32_t len = GRPC_CHTTP2_VARINT_LENGTH(c->table->max_size(), 3);
GRPC_CHTTP2_WRITE_VARINT(c->table->max_size(), 3, 0x20,
add_tiny_header_data(st, len), len);
c->advertise_table_size_change = 0;
void HPackCompressor::Framer::EmitLitHdrWithStringKeyNotIdx(grpc_mdelem elem) {
GRPC_STATS_INC_HPACK_SEND_LITHDR_NOTIDX_V();
GRPC_STATS_INC_HPACK_SEND_UNCOMPRESSED();
StringKey key(GRPC_MDKEY(elem));
key.WritePrefix(0x00, AddTiny(key.prefix_length()));
Add(grpc_slice_ref_internal(key.key()));
StringValue emit(UnsureIfInterned(), elem, use_true_binary_metadata_);
emit.WritePrefix(AddTiny(emit.prefix_length()));
Add(emit.data());
}
static void GPR_ATTRIBUTE_NOINLINE hpack_enc_log(grpc_mdelem elem) {
void HPackCompressor::Framer::AdvertiseTableSizeChange() {
VarintWriter<3> w(compressor_->table_.max_size());
w.Write(0x20, AddTiny(w.length()));
}
void HPackCompressor::Framer::Log(grpc_mdelem elem) {
char* k = grpc_slice_to_c_string(GRPC_MDKEY(elem));
char* v = nullptr;
if (grpc_is_binary_header_internal(GRPC_MDKEY(elem))) {
@ -401,154 +361,113 @@ struct EmitIndexedStatus {
const bool can_add = false;
};
static EmitIndexedStatus maybe_emit_indexed(grpc_chttp2_hpack_compressor* c,
grpc_mdelem elem,
framer_state* st) {
const uint32_t elem_hash =
GRPC_MDELEM_STORAGE(elem) == GRPC_MDELEM_STORAGE_INTERNED
? reinterpret_cast<grpc_core::InternedMetadata*>(
GRPC_MDELEM_DATA(elem))
->hash()
: reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(elem))
->hash();
// Update filter to see if we can perhaps add this elem.
bool can_add_to_hashtable =
c->filter_elems->AddElement(HASH_FRAGMENT_1(elem_hash));
/* is this elem currently in the decoders table? */
auto indices_key = c->elem_index->Lookup(
grpc_chttp2_hpack_compressor::KeyElem(elem, elem_hash));
if (indices_key.has_value() &&
c->table->ConvertableToDynamicIndex(*indices_key)) {
emit_indexed(c, c->table->DynamicIndex(*indices_key), st);
return EmitIndexedStatus(elem_hash, true, false);
}
/* Didn't hit either cuckoo index, so no emit. */
return EmitIndexedStatus(elem_hash, false, can_add_to_hashtable);
}
static void emit_maybe_add(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
framer_state* st, uint32_t indices_key,
bool should_add_elem, size_t decoder_space_usage,
uint32_t elem_hash, uint32_t key_hash) {
if (should_add_elem) {
emit_lithdr<EmitLitHdrType::INC_IDX>(c, c->table->DynamicIndex(indices_key),
elem, st);
add_elem(c, elem, decoder_space_usage, elem_hash, key_hash);
} else {
emit_lithdr<EmitLitHdrType::NO_IDX>(c, c->table->DynamicIndex(indices_key),
elem, st);
}
}
/* encode an mdelem */
static void hpack_enc(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
framer_state* st) {
void HPackCompressor::Framer::EncodeDynamic(grpc_mdelem elem) {
const grpc_slice& elem_key = GRPC_MDKEY(elem);
/* User-provided key len validated in grpc_validate_header_key_is_legal(). */
// User-provided key len validated in grpc_validate_header_key_is_legal().
GPR_DEBUG_ASSERT(GRPC_SLICE_LENGTH(elem_key) > 0);
/* Header ordering: all reserved headers (prefixed with ':') must precede
* regular headers. This can be a debug assert, since:
* 1) User cannot give us ':' headers (grpc_validate_header_key_is_legal()).
* 2) grpc filters/core should be checked during debug builds. */
// Header ordering: all reserved headers (prefixed with ':') must precede
// regular headers. This can be a debug assert, since:
// 1) User cannot give us ':' headers (grpc_validate_header_key_is_legal()).
// 2) grpc filters/core should be checked during debug builds. */
#ifndef NDEBUG
if (GRPC_SLICE_START_PTR(elem_key)[0] != ':') { /* regular header */
st->seen_regular_header = 1;
seen_regular_header_ = true;
} else {
GPR_DEBUG_ASSERT(
st->seen_regular_header == 0 &&
!seen_regular_header_ &&
"Reserved header (colon-prefixed) happening after regular ones.");
}
#endif
if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
hpack_enc_log(elem);
Log(elem);
}
const bool elem_interned = GRPC_MDELEM_IS_INTERNED(elem);
const bool key_interned = elem_interned || grpc_slice_is_interned(elem_key);
/* Key is not interned, emit literals. */
// Key is not interned, emit literals.
if (!key_interned) {
emit_lithdr_v<EmitLitHdrVType::NO_IDX_V>(c, elem, st);
EmitLitHdrWithStringKeyNotIdx(elem);
return;
}
/* Interned metadata => maybe already indexed. */
const EmitIndexedStatus ret =
elem_interned ? maybe_emit_indexed(c, elem, st) : EmitIndexedStatus();
if (ret.emitted) {
return;
uint32_t elem_hash = 0;
if (elem_interned) {
// Update filter to see if we can perhaps add this elem.
elem_hash = GRPC_MDELEM_STORAGE(elem) == GRPC_MDELEM_STORAGE_INTERNED
? reinterpret_cast<grpc_core::InternedMetadata*>(
GRPC_MDELEM_DATA(elem))
->hash()
: reinterpret_cast<grpc_core::StaticMetadata*>(
GRPC_MDELEM_DATA(elem))
->hash();
bool can_add_to_hashtable =
compressor_->filter_elems_.AddElement(elem_hash % kNumFilterValues);
/* is this elem currently in the decoders table? */
auto indices_key =
compressor_->elem_index_.Lookup(KeyElem(elem, elem_hash));
if (indices_key.has_value() &&
compressor_->table_.ConvertableToDynamicIndex(*indices_key)) {
EmitIndexed(compressor_->table_.DynamicIndex(*indices_key));
return;
}
/* Didn't hit either cuckoo index, so no emit. */
if (!can_add_to_hashtable) elem_hash = 0;
}
/* should this elem be in the table? */
const size_t decoder_space_usage =
grpc_core::MetadataSizeInHPackTable(elem, st->use_true_binary_metadata);
grpc_core::MetadataSizeInHPackTable(elem, use_true_binary_metadata_);
const bool decoder_space_available =
decoder_space_usage < kMaxDecoderSpaceUsage;
const bool should_add_elem =
elem_interned && decoder_space_available && ret.can_add;
const uint32_t elem_hash = ret.elem_hash;
elem_interned && decoder_space_available && elem_hash != 0;
/* no hits for the elem... maybe there's a key? */
const uint32_t key_hash = elem_key.refcount->Hash(elem_key);
auto indices_key = c->key_index->Lookup(
grpc_chttp2_hpack_compressor::KeySliceRef(elem_key.refcount, key_hash));
auto indices_key =
compressor_->key_index_.Lookup(KeySliceRef(elem_key.refcount, key_hash));
if (indices_key.has_value() &&
c->table->ConvertableToDynamicIndex(*indices_key)) {
emit_maybe_add(c, elem, st, *indices_key, should_add_elem,
decoder_space_usage, elem_hash, key_hash);
compressor_->table_.ConvertableToDynamicIndex(*indices_key)) {
if (should_add_elem) {
EmitLitHdrIncIdx(compressor_->table_.DynamicIndex(*indices_key), elem);
compressor_->AddElem(elem, decoder_space_usage, elem_hash, key_hash);
} else {
EmitLitHdrNotIdx(compressor_->table_.DynamicIndex(*indices_key), elem);
}
return;
}
/* no elem, key in the table... fall back to literal emission */
const bool should_add_key = !elem_interned && decoder_space_available;
if (should_add_elem || should_add_key) {
emit_lithdr_v<EmitLitHdrVType::INC_IDX_V>(c, elem, st);
EmitLitHdrWithStringKeyIncIdx(elem);
} else {
emit_lithdr_v<EmitLitHdrVType::NO_IDX_V>(c, elem, st);
EmitLitHdrWithStringKeyNotIdx(elem);
}
if (should_add_elem) {
add_elem(c, elem, decoder_space_usage, elem_hash, key_hash);
compressor_->AddElem(elem, decoder_space_usage, elem_hash, key_hash);
} else if (should_add_key) {
add_key(c, elem, decoder_space_usage, key_hash);
compressor_->AddKey(elem, decoder_space_usage, key_hash);
}
}
#define STRLEN_LIT(x) (sizeof(x) - 1)
#define TIMEOUT_KEY "grpc-timeout"
static void deadline_enc(grpc_chttp2_hpack_compressor* c, grpc_millis deadline,
framer_state* st) {
void HPackCompressor::Framer::EncodeDeadline(grpc_millis deadline) {
char timeout_str[GRPC_HTTP2_TIMEOUT_ENCODE_MIN_BUFSIZE];
grpc_mdelem mdelem;
grpc_http2_encode_timeout(deadline - grpc_core::ExecCtx::Get()->Now(),
timeout_str);
mdelem = grpc_mdelem_from_slices(
GRPC_MDSTR_GRPC_TIMEOUT, grpc_core::UnmanagedMemorySlice(timeout_str));
hpack_enc(c, mdelem, st);
EncodeDynamic(mdelem);
GRPC_MDELEM_UNREF(mdelem);
}
void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor* c) {
c->advertise_table_size_change = 0;
c->max_usable_size = grpc_core::hpack_constants::kInitialTableSize;
c->table.Init();
c->filter_elems.Init();
c->elem_index.Init();
c->key_index.Init();
void HPackCompressor::SetMaxUsableSize(uint32_t max_table_size) {
max_usable_size_ = max_table_size;
SetMaxTableSize(std::min(table_.max_size(), max_table_size));
}
void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor* c) {
c->elem_index.Destroy();
c->filter_elems.Destroy();
c->key_index.Destroy();
c->table.Destroy();
}
void grpc_chttp2_hpack_compressor_set_max_usable_size(
grpc_chttp2_hpack_compressor* c, uint32_t max_table_size) {
c->max_usable_size = max_table_size;
grpc_chttp2_hpack_compressor_set_max_table_size(
c, std::min(c->table->max_size(), max_table_size));
}
void grpc_chttp2_hpack_compressor_set_max_table_size(
grpc_chttp2_hpack_compressor* c, uint32_t max_table_size) {
if (c->table->SetMaxSize(std::min(c->max_usable_size, max_table_size))) {
c->advertise_table_size_change = 1;
void HPackCompressor::SetMaxTableSize(uint32_t max_table_size) {
if (table_.SetMaxSize(std::min(max_usable_size_, max_table_size))) {
advertise_table_size_change_ = true;
if (GRPC_TRACE_FLAG_ENABLED(grpc_http_trace)) {
gpr_log(GPR_INFO, "set max table size from encoder to %d",
max_table_size);
@ -556,72 +475,33 @@ void grpc_chttp2_hpack_compressor_set_max_table_size(
}
}
void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c,
grpc_mdelem** extra_headers,
size_t extra_headers_size,
grpc_metadata_batch* metadata,
const grpc_encode_header_options* options,
grpc_slice_buffer* outbuf) {
/* grpc_chttp2_encode_header is called by FlushInitial/TrailingMetadata in
writing.cc. Specifically, on streams returned by NextStream(), which
returns streams from the list GRPC_CHTTP2_LIST_WRITABLE. The only way to be
added to the list is via grpc_chttp2_list_add_writable_stream(), which
validates that stream_id is not 0. So, this can be a debug assert. */
GPR_DEBUG_ASSERT(options->stream_id != 0);
framer_state st;
#ifndef NDEBUG
st.seen_regular_header = 0;
#endif
st.stream_id = options->stream_id;
st.output = outbuf;
st.is_first_frame = 1;
st.stats = options->stats;
st.max_frame_size = options->max_frame_size;
st.use_true_binary_metadata = options->use_true_binary_metadata;
st.is_end_of_stream = options->is_eof;
/* Encode a metadata batch; store the returned values, representing
a metadata element that needs to be unreffed back into the metadata
slot. THIS MAY NOT BE THE SAME ELEMENT (if a decoder table slot got
updated). After this loop, we'll do a batch unref of elements. */
begin_frame(&st);
if (c->advertise_table_size_change != 0) {
emit_advertise_table_size_change(c, &st);
HPackCompressor::Framer::Framer(const EncodeHeaderOptions& options,
HPackCompressor* compressor,
grpc_slice_buffer* output)
: max_frame_size_(options.max_frame_size),
use_true_binary_metadata_(options.use_true_binary_metadata),
is_end_of_stream_(options.is_end_of_stream),
stream_id_(options.stream_id),
output_(output),
stats_(options.stats),
compressor_(compressor),
prefix_(BeginFrame()) {
if (absl::exchange(compressor_->advertise_table_size_change_, false)) {
AdvertiseTableSizeChange();
}
for (size_t i = 0; i < extra_headers_size; ++i) {
grpc_mdelem md = *extra_headers[i];
const bool is_static =
GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC;
uintptr_t static_index;
if (is_static &&
(static_index =
reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(md))
->StaticIndex()) <
grpc_core::hpack_constants::kLastStaticEntry) {
emit_indexed(c, static_cast<uint32_t>(static_index + 1), &st);
} else {
hpack_enc(c, md, &st);
}
}
grpc_metadata_batch_assert_ok(metadata);
for (grpc_linked_mdelem* l = metadata->list.head; l; l = l->next) {
const bool is_static =
GRPC_MDELEM_STORAGE(l->md) == GRPC_MDELEM_STORAGE_STATIC;
uintptr_t static_index;
if (is_static &&
(static_index = reinterpret_cast<grpc_core::StaticMetadata*>(
GRPC_MDELEM_DATA(l->md))
->StaticIndex()) <
grpc_core::hpack_constants::kLastStaticEntry) {
emit_indexed(c, static_cast<uint32_t>(static_index + 1), &st);
} else {
hpack_enc(c, l->md, &st);
}
void HPackCompressor::Framer::Encode(grpc_mdelem md) {
if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) {
const uintptr_t static_index =
reinterpret_cast<grpc_core::StaticMetadata*>(GRPC_MDELEM_DATA(md))
->StaticIndex();
if (static_index < hpack_constants::kLastStaticEntry) {
EmitIndexed(static_cast<uint32_t>(static_index + 1));
return;
}
}
grpc_millis deadline = metadata->deadline;
if (deadline != GRPC_MILLIS_INF_FUTURE) {
deadline_enc(c, deadline, &st);
}
finish_frame(&st, 1);
EncodeDynamic(md);
}
} // namespace grpc_core

208
src/core/ext/transport/chttp2/transport/hpack_encoder.h
Unescape View File

@ -31,33 +31,162 @@
#include "src/core/lib/transport/metadata_batch.h"
#include "src/core/lib/transport/transport.h"
// This should be <= 8. We use 6 to save space.
#define GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS 6
#define GRPC_CHTTP2_HPACKC_NUM_VALUES (1 << GRPC_CHTTP2_HPACKC_NUM_VALUES_BITS)
/* initial table size, per spec */
#define GRPC_CHTTP2_HPACKC_INITIAL_TABLE_SIZE 4096
/* maximum table size we'll actually use */
#define GRPC_CHTTP2_HPACKC_MAX_TABLE_SIZE (1024 * 1024)
extern grpc_core::TraceFlag grpc_http_trace;
struct grpc_chttp2_hpack_compressor {
namespace grpc_core {
// Wrapper to take an array of mdelems and make them encodable
class MetadataArray {
public:
MetadataArray(grpc_mdelem** elems, size_t count)
: elems_(elems), count_(count) {}
template <typename Encoder>
void Encode(Encoder* encoder) const {
for (size_t i = 0; i < count_; i++) {
encoder->Encode(*elems_[i]);
}
}
private:
grpc_mdelem** elems_;
size_t count_;
};
namespace metadata_detail {
template <typename A, typename B>
class ConcatMetadata {
public:
ConcatMetadata(const A& a, const B& b) : a_(a), b_(b) {}
template <typename Encoder>
void Encode(Encoder* encoder) const {
a_.Encode(encoder);
b_.Encode(encoder);
}
private:
const A& a_;
const B& b_;
};
} // namespace metadata_detail
template <typename A, typename B>
metadata_detail::ConcatMetadata<A, B> ConcatMetadata(const A& a, const B& b) {
return metadata_detail::ConcatMetadata<A, B>(a, b);
}
class HPackCompressor {
public:
HPackCompressor() = default;
~HPackCompressor() = default;
// Maximum table size we'll actually use.
static constexpr uint32_t kMaxTableSize = 1024 * 1024;
void SetMaxTableSize(uint32_t max_table_size);
void SetMaxUsableSize(uint32_t max_table_size);
uint32_t test_only_table_size() const {
return table_.test_only_table_size();
}
struct EncodeHeaderOptions {
uint32_t stream_id;
bool is_end_of_stream;
bool use_true_binary_metadata;
size_t max_frame_size;
grpc_transport_one_way_stats* stats;
};
template <typename HeaderSet>
void EncodeHeaders(const EncodeHeaderOptions& options,
const HeaderSet& headers, grpc_slice_buffer* output) {
Framer framer(options, this, output);
headers.Encode(&framer);
}
class Framer {
public:
Framer(const EncodeHeaderOptions& options, HPackCompressor* compressor,
grpc_slice_buffer* output);
~Framer() { FinishFrame(true); }
Framer(const Framer&) = delete;
Framer& operator=(const Framer&) = delete;
void Encode(grpc_mdelem md);
void EncodeDeadline(grpc_millis deadline);
private:
struct FramePrefix {
// index (in output_) of the header for the frame
size_t header_idx;
// number of bytes in 'output' when we started the frame - used to
// calculate frame length
size_t output_length_at_start_of_frame;
};
FramePrefix BeginFrame();
void FinishFrame(bool is_header_boundary);
void EnsureSpace(size_t need_bytes);
void AdvertiseTableSizeChange();
void EmitIndexed(uint32_t index);
void EncodeDynamic(grpc_mdelem elem);
static GPR_ATTRIBUTE_NOINLINE void Log(grpc_mdelem elem);
void EmitLitHdrIncIdx(uint32_t key_index, grpc_mdelem elem);
void EmitLitHdrNotIdx(uint32_t key_index, grpc_mdelem elem);
void EmitLitHdrWithStringKeyIncIdx(grpc_mdelem elem);
void EmitLitHdrWithStringKeyNotIdx(grpc_mdelem elem);
size_t CurrentFrameSize() const;
void Add(grpc_slice slice);
uint8_t* AddTiny(size_t len);
// maximum size of a frame
const size_t max_frame_size_;
bool is_first_frame_ = true;
const bool use_true_binary_metadata_;
const bool is_end_of_stream_;
// output stream id
const uint32_t stream_id_;
#ifndef NDEBUG
// have we seen a regular (non-colon-prefixed) header yet?
bool seen_regular_header_ = false;
#endif
grpc_slice_buffer* const output_;
grpc_transport_one_way_stats* const stats_;
HPackCompressor* const compressor_;
FramePrefix prefix_;
};
private:
static constexpr size_t kNumFilterValues = 64;
void AddKeyWithIndex(grpc_slice_refcount* key_ref, uint32_t new_index,
uint32_t key_hash);
void AddElemWithIndex(grpc_mdelem elem, uint32_t new_index,
uint32_t elem_hash, uint32_t key_hash);
void AddElem(grpc_mdelem elem, size_t elem_size, uint32_t elem_hash,
uint32_t key_hash);
void AddKey(grpc_mdelem elem, size_t elem_size, uint32_t key_hash);
// maximum number of bytes we'll use for the decode table (to guard against
// peers ooming us by setting decode table size high)
uint32_t max_usable_size;
grpc_core::ManualConstructor<grpc_core::HPackEncoderTable> table;
/** if non-zero, advertise to the decoder that we'll start using a table
of this size */
uint8_t advertise_table_size_change;
/* filter tables for elems: this tables provides an approximate
popularity count for particular hashes, and are used to determine whether
a new literal should be added to the compression table or not.
They track a single integer that counts how often a particular value has
been seen. When that count reaches max (255), all values are halved. */
grpc_core::ManualConstructor<
grpc_core::PopularityCount<GRPC_CHTTP2_HPACKC_NUM_VALUES>>
filter_elems;
uint32_t max_usable_size_ = hpack_constants::kInitialTableSize;
// if non-zero, advertise to the decoder that we'll start using a table
// of this size
bool advertise_table_size_change_ = false;
HPackEncoderTable table_;
// filter tables for elems: this tables provides an approximate
// popularity count for particular hashes, and are used to determine whether
// a new literal should be added to the compression table or not.
// They track a single integer that counts how often a particular value has
// been seen. When that count reaches max (255), all values are halved.
grpc_core::PopularityCount<kNumFilterValues> filter_elems_;
class KeyElem {
public:
@ -132,35 +261,12 @@ struct grpc_chttp2_hpack_compressor {
uint32_t hash_;
};
/* entry tables for keys & elems: these tables track values that have been
seen and *may* be in the decompressor table */
grpc_core::ManualConstructor<
grpc_core::HPackEncoderIndex<KeyElem, GRPC_CHTTP2_HPACKC_NUM_VALUES>>
elem_index;
grpc_core::ManualConstructor<
grpc_core::HPackEncoderIndex<KeySliceRef, GRPC_CHTTP2_HPACKC_NUM_VALUES>>
key_index;
// entry tables for keys & elems: these tables track values that have been
// seen and *may* be in the decompressor table
grpc_core::HPackEncoderIndex<KeyElem, kNumFilterValues> elem_index_;
grpc_core::HPackEncoderIndex<KeySliceRef, kNumFilterValues> key_index_;
};
void grpc_chttp2_hpack_compressor_init(grpc_chttp2_hpack_compressor* c);
void grpc_chttp2_hpack_compressor_destroy(grpc_chttp2_hpack_compressor* c);
void grpc_chttp2_hpack_compressor_set_max_table_size(
grpc_chttp2_hpack_compressor* c, uint32_t max_table_size);
void grpc_chttp2_hpack_compressor_set_max_usable_size(
grpc_chttp2_hpack_compressor* c, uint32_t max_table_size);
struct grpc_encode_header_options {
uint32_t stream_id;
bool is_eof;
bool use_true_binary_metadata;
size_t max_frame_size;
grpc_transport_one_way_stats* stats;
};
void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c,
grpc_mdelem** extra_headers,
size_t extra_headers_size,
grpc_metadata_batch* metadata,
const grpc_encode_header_options* options,
grpc_slice_buffer* outbuf);
} // namespace grpc_core
#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_ENCODER_H */

2
src/core/ext/transport/chttp2/transport/internal.h
Unescape View File

@ -345,7 +345,7 @@ struct grpc_chttp2_transport {
/** data to write now */
grpc_slice_buffer outbuf;
/** hpack encoding */
grpc_chttp2_hpack_compressor hpack_compressor;
grpc_core::HPackCompressor hpack_compressor;
/** is this a client? */
bool is_client;

10
src/core/ext/transport/chttp2/transport/varint.cc
Unescape View File

@ -22,7 +22,9 @@
#include "absl/base/attributes.h"
uint32_t grpc_chttp2_hpack_varint_length(uint32_t tail_value) {
namespace grpc_core {
uint32_t VarintLength(uint32_t tail_value) {
if (tail_value < (1 << 7)) {
return 2;
} else if (tail_value < (1 << 14)) {
@ -36,8 +38,8 @@ uint32_t grpc_chttp2_hpack_varint_length(uint32_t tail_value) {
}
}
void grpc_chttp2_hpack_write_varint_tail(uint32_t tail_value, uint8_t* target,
uint32_t tail_length) {
void VarintWriteTail(uint32_t tail_value, uint8_t* target,
uint32_t tail_length) {
switch (tail_length) {
case 5:
target[4] = static_cast<uint8_t>((tail_value >> 28) | 0x80);
@ -56,3 +58,5 @@ void grpc_chttp2_hpack_write_varint_tail(uint32_t tail_value, uint8_t* target,
}
target[tail_length - 1] &= 0x7f;
}
} // namespace grpc_core

67
src/core/ext/transport/chttp2/transport/varint.h
Unescape View File

@ -23,38 +23,49 @@
/* Helpers for hpack varint encoding */
namespace grpc_core {
/* maximum value that can be bitpacked with the opcode if the opcode has a
prefix of length prefix_bits */
constexpr uint32_t MaxInVarintPrefix(uint8_t prefix_bits) {
return (1 << (8 - prefix_bits)) - 1;
}
/* length of a value that needs varint tail encoding (it's bigger than can be
bitpacked into the opcode byte) - returned value includes the length of the
opcode byte */
uint32_t grpc_chttp2_hpack_varint_length(uint32_t tail_value);
uint32_t VarintLength(uint32_t tail_value);
void VarintWriteTail(uint32_t tail_value, uint8_t* target,
uint32_t tail_length);
void grpc_chttp2_hpack_write_varint_tail(uint32_t tail_value, uint8_t* target,
uint32_t tail_length);
template <uint8_t kPrefixBits>
class VarintWriter {
public:
static constexpr uint32_t kMaxInPrefix = MaxInVarintPrefix(kPrefixBits);
/* maximum value that can be bitpacked with the opcode if the opcode has a
prefix
of length prefix_bits */
#define GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits) \
((uint32_t)((1 << (8 - (prefix_bits))) - 1))
/* length required to bitpack a value */
#define GRPC_CHTTP2_VARINT_LENGTH(n, prefix_bits) \
((n) < GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits) \
? 1u \
: grpc_chttp2_hpack_varint_length( \
(n)-GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits)))
#define GRPC_CHTTP2_WRITE_VARINT(n, prefix_bits, prefix_or, target, length) \
do { \
uint8_t* tgt = target; \
if ((length) == 1u) { \
(tgt)[0] = (uint8_t)((prefix_or) | (n)); \
} else { \
(tgt)[0] = \
(prefix_or) | (uint8_t)GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits); \
grpc_chttp2_hpack_write_varint_tail( \
(n)-GRPC_CHTTP2_MAX_IN_PREFIX(prefix_bits), (tgt) + 1, (length)-1); \
} \
} while (0)
explicit VarintWriter(uint32_t value)
: value_(value),
length_(value < kMaxInPrefix ? 1 : VarintLength(value - kMaxInPrefix)) {
}
uint32_t value() const { return value_; }
uint32_t length() const { return length_; }
void Write(uint8_t prefix, uint8_t* target) const {
if (length_ == 1) {
target[0] = prefix | value_;
} else {
target[0] = prefix | kMaxInPrefix;
VarintWriteTail(value_ - kMaxInPrefix, target + 1, length_ - 1);
}
}
private:
const uint32_t value_;
// length required to bitpack value_
const uint32_t length_;
};
} // namespace grpc_core
#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_VARINT_H */

53
src/core/ext/transport/chttp2/transport/writing.cc
Unescape View File

@ -259,8 +259,7 @@ class WriteContext {
}
void EnactHpackSettings() {
grpc_chttp2_hpack_compressor_set_max_table_size(
&t_->hpack_compressor,
t_->hpack_compressor.SetMaxTableSize(
t_->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_HEADER_TABLE_SIZE]);
}
@ -457,18 +456,20 @@ class StreamWriteContext {
is_default_initial_metadata(s_->send_initial_metadata)) {
ConvertInitialMetadataToTrailingMetadata();
} else {
grpc_encode_header_options hopt = {
s_->id, // stream_id
false, // is_eof
t_->settings[GRPC_PEER_SETTINGS]
t_->hpack_compressor.EncodeHeaders(
grpc_core::HPackCompressor::EncodeHeaderOptions{
s_->id, // stream_id
false, // is_eof
t_->settings
[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA] !=
0, // use_true_binary_metadata
t_->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], // max_frame_size
&s_->stats.outgoing // stats
};
grpc_chttp2_encode_header(&t_->hpack_compressor, nullptr, 0,
s_->send_initial_metadata, &hopt, &t_->outbuf);
0, // use_true_binary_metadata
t_->settings
[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE], // max_frame_size
&s_->stats.outgoing // stats
},
*s_->send_initial_metadata, &t_->outbuf);
grpc_chttp2_reset_ping_clock(t_);
write_context_->IncInitialMetadataWrites();
}
@ -565,18 +566,22 @@ class StreamWriteContext {
grpc_chttp2_encode_data(s_->id, &s_->flow_controlled_buffer, 0, true,
&s_->stats.outgoing, &t_->outbuf);
} else {
grpc_encode_header_options hopt = {
s_->id, true,
t_->settings[GRPC_PEER_SETTINGS]
t_->hpack_compressor.EncodeHeaders(
grpc_core::HPackCompressor::EncodeHeaderOptions{
s_->id, true,
t_->settings
[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_GRPC_ALLOW_TRUE_BINARY_METADATA] !=
0,
t_->settings[GRPC_PEER_SETTINGS][GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE],
&s_->stats.outgoing};
grpc_chttp2_encode_header(&t_->hpack_compressor,
extra_headers_for_trailing_metadata_,
num_extra_headers_for_trailing_metadata_,
s_->send_trailing_metadata, &hopt, &t_->outbuf);
0,
t_->settings[GRPC_PEER_SETTINGS]
[GRPC_CHTTP2_SETTINGS_MAX_FRAME_SIZE],
&s_->stats.outgoing},
grpc_core::ConcatMetadata(
grpc_core::MetadataArray(
extra_headers_for_trailing_metadata_,
num_extra_headers_for_trailing_metadata_),
*s_->send_trailing_metadata),
&t_->outbuf);
}
write_context_->IncTrailingMetadataWrites();
grpc_chttp2_reset_ping_clock(t_);

8
src/core/lib/transport/metadata_batch.h
Unescape View File

@ -57,6 +57,14 @@ typedef struct grpc_metadata_batch {
or GRPC_MILLIS_INF_FUTURE if this batch does not need to send a
grpc-timeout */
grpc_millis deadline;
template <typename Encoder>
void Encode(Encoder* encoder) const {
for (auto* l = list.head; l; l = l->next) {
encoder->Encode(l->md);
}
if (deadline != GRPC_MILLIS_INF_FUTURE) encoder->EncodeDeadline(deadline);
}
} grpc_metadata_batch;
void grpc_metadata_batch_init(grpc_metadata_batch* batch);

33
test/core/transport/chttp2/hpack_encoder_test.cc
Unescape View File

@ -41,7 +41,7 @@
#define TEST(x) run_test(x, #x)
grpc_chttp2_hpack_compressor g_compressor;
grpc_core::HPackCompressor* g_compressor;
int g_failure = 0;
void** to_delete = nullptr;
@ -199,14 +199,14 @@ static void verify(const verify_params params, const char* expected,
grpc_transport_one_way_stats stats;
stats = {};
grpc_encode_header_options hopt = {
grpc_core::HPackCompressor::EncodeHeaderOptions hopt{
0xdeadbeef, /* stream_id */
params.eof, /* is_eof */
params.use_true_binary_metadata, /* use_true_binary_metadata */
16384, /* max_frame_size */
&stats /* stats */
};
grpc_chttp2_encode_header(&g_compressor, nullptr, 0, &b, &hopt, &output);
g_compressor->EncodeHeaders(hopt, b, &output);
verify_frames(output, params.eof);
merged = grpc_slice_merge(output.slices, output.count);
grpc_slice_buffer_destroy_internal(&output);
@ -273,12 +273,13 @@ static void verify_continuation_headers(const char* key, const char* value,
grpc_transport_one_way_stats stats;
stats = {};
grpc_encode_header_options hopt = {0xdeadbeef, /* stream_id */
is_eof, /* is_eof */
false, /* use_true_binary_metadata */
150, /* max_frame_size */
&stats /* stats */};
grpc_chttp2_encode_header(&g_compressor, nullptr, 0, &b, &hopt, &output);
grpc_core::HPackCompressor::EncodeHeaderOptions hopt = {
0xdeadbeef, /* stream_id */
is_eof, /* is_eof */
false, /* use_true_binary_metadata */
150, /* max_frame_size */
&stats /* stats */};
g_compressor->EncodeHeaders(hopt, b, &output);
verify_frames(output, is_eof);
grpc_slice_buffer_destroy_internal(&output);
grpc_metadata_batch_destroy(&b);
@ -307,7 +308,7 @@ static void encode_int_to_str(int i, char* p) {
static void test_decode_table_overflow() {
// Decrease the default table size to make decode table overflow easier.
grpc_chttp2_hpack_compressor_set_max_table_size(&g_compressor, 1024);
g_compressor->SetMaxTableSize(1024);
int i;
char key[3], value[3];
@ -347,7 +348,7 @@ static void verify_table_size_change_match_elem_size(const char* key,
grpc_slice_intern(grpc_slice_from_static_string(key)),
grpc_slice_intern(grpc_slice_from_static_string(value)));
size_t elem_size = grpc_core::MetadataSizeInHPackTable(elem, use_true_binary);
size_t initial_table_size = g_compressor.table->test_only_table_size();
size_t initial_table_size = g_compressor->test_only_table_size();
grpc_linked_mdelem* e =
static_cast<grpc_linked_mdelem*>(gpr_malloc(sizeof(*e)));
grpc_metadata_batch b;
@ -362,18 +363,18 @@ static void verify_table_size_change_match_elem_size(const char* key,
grpc_transport_one_way_stats stats;
stats = {};
grpc_encode_header_options hopt = {
grpc_core::HPackCompressor::EncodeHeaderOptions hopt = {
0xdeadbeef, /* stream_id */
false, /* is_eof */
use_true_binary, /* use_true_binary_metadata */
16384, /* max_frame_size */
&stats /* stats */};
grpc_chttp2_encode_header(&g_compressor, nullptr, 0, &b, &hopt, &output);
g_compressor->EncodeHeaders(hopt, b, &output);
verify_frames(output, false);
grpc_slice_buffer_destroy_internal(&output);
grpc_metadata_batch_destroy(&b);
GPR_ASSERT(g_compressor.table->test_only_table_size() ==
GPR_ASSERT(g_compressor->test_only_table_size() ==
elem_size + initial_table_size);
gpr_free(e);
}
@ -399,9 +400,9 @@ static void test_interned_key_indexed() {
static void run_test(void (*test)(), const char* name) {
gpr_log(GPR_INFO, "RUN TEST: %s", name);
grpc_core::ExecCtx exec_ctx;
grpc_chttp2_hpack_compressor_init(&g_compressor);
g_compressor = new grpc_core::HPackCompressor();
test();
grpc_chttp2_hpack_compressor_destroy(&g_compressor);
delete g_compressor;
}
int main(int argc, char** argv) {

14
test/core/transport/chttp2/varint_test.cc
Unescape View File

@ -24,24 +24,24 @@
#include "test/core/util/test_config.h"
static void test_varint(uint32_t value, uint32_t prefix_bits, uint8_t prefix_or,
template <uint8_t kPrefixBits>
static void test_varint(uint32_t value, uint8_t prefix_or,
const char* expect_bytes, size_t expect_length) {
uint32_t nbytes = GRPC_CHTTP2_VARINT_LENGTH(value, prefix_bits);
grpc_core::VarintWriter<kPrefixBits> w(value);
grpc_slice expect =
grpc_slice_from_copied_buffer(expect_bytes, expect_length);
grpc_slice slice;
gpr_log(GPR_DEBUG, "Test: 0x%08x", value);
GPR_ASSERT(nbytes == expect_length);
slice = grpc_slice_malloc(nbytes);
GRPC_CHTTP2_WRITE_VARINT(value, prefix_bits, prefix_or,
GRPC_SLICE_START_PTR(slice), nbytes);
GPR_ASSERT(w.length() == expect_length);
slice = grpc_slice_malloc(w.length());
w.Write(prefix_or, GRPC_SLICE_START_PTR(slice));
GPR_ASSERT(grpc_slice_eq(expect, slice));
grpc_slice_unref(expect);
grpc_slice_unref(slice);
}
#define TEST_VARINT(value, prefix_bits, prefix_or, expect) \
test_varint(value, prefix_bits, prefix_or, expect, sizeof(expect) - 1)
test_varint<prefix_bits>(value, prefix_or, expect, sizeof(expect) - 1)
int main(int argc, char** argv) {
grpc::testing::TestEnvironment env(argc, argv);

49
test/cpp/microbenchmarks/bm_chttp2_hpack.cc
Unescape View File

@ -53,11 +53,8 @@ static grpc_slice MakeSlice(std::vector<uint8_t> bytes) {
static void BM_HpackEncoderInitDestroy(benchmark::State& state) {
TrackCounters track_counters;
grpc_core::ExecCtx exec_ctx;
std::unique_ptr<grpc_chttp2_hpack_compressor> c(
new grpc_chttp2_hpack_compressor);
for (auto _ : state) {
grpc_chttp2_hpack_compressor_init(c.get());
grpc_chttp2_hpack_compressor_destroy(c.get());
grpc_core::HPackCompressor c;
grpc_core::ExecCtx::Get()->Flush();
}
@ -74,27 +71,25 @@ static void BM_HpackEncoderEncodeDeadline(benchmark::State& state) {
grpc_metadata_batch_init(&b);
b.deadline = saved_now + 30 * 1000;
std::unique_ptr<grpc_chttp2_hpack_compressor> c(
new grpc_chttp2_hpack_compressor);
grpc_chttp2_hpack_compressor_init(c.get());
grpc_core::HPackCompressor c;
grpc_transport_one_way_stats stats;
stats = {};
grpc_slice_buffer outbuf;
grpc_slice_buffer_init(&outbuf);
while (state.KeepRunning()) {
grpc_encode_header_options hopt = {
static_cast<uint32_t>(state.iterations()),
true,
false,
static_cast<size_t>(1024),
&stats,
};
grpc_chttp2_encode_header(c.get(), nullptr, 0, &b, &hopt, &outbuf);
c.EncodeHeaders(
grpc_core::HPackCompressor::EncodeHeaderOptions{
static_cast<uint32_t>(state.iterations()),
true,
false,
static_cast<size_t>(1024),
&stats,
},
b, &outbuf);
grpc_slice_buffer_reset_and_unref_internal(&outbuf);
grpc_core::ExecCtx::Get()->Flush();
}
grpc_metadata_batch_destroy(&b);
grpc_chttp2_hpack_compressor_destroy(c.get());
grpc_slice_buffer_destroy_internal(&outbuf);
std::ostringstream label;
@ -124,23 +119,22 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State& state) {
"addmd", grpc_metadata_batch_add_tail(&b, &storage[i], elems[i])));
}
std::unique_ptr<grpc_chttp2_hpack_compressor> c(
new grpc_chttp2_hpack_compressor);
grpc_chttp2_hpack_compressor_init(c.get());
grpc_core::HPackCompressor c;
grpc_transport_one_way_stats stats;
stats = {};
grpc_slice_buffer outbuf;
grpc_slice_buffer_init(&outbuf);
while (state.KeepRunning()) {
static constexpr int kEnsureMaxFrameAtLeast = 2;
grpc_encode_header_options hopt = {
static_cast<uint32_t>(state.iterations()),
state.range(0) != 0,
Fixture::kEnableTrueBinary,
static_cast<size_t>(state.range(1) + kEnsureMaxFrameAtLeast),
&stats,
};
grpc_chttp2_encode_header(c.get(), nullptr, 0, &b, &hopt, &outbuf);
c.EncodeHeaders(
grpc_core::HPackCompressor::EncodeHeaderOptions{
static_cast<uint32_t>(state.iterations()),
state.range(0) != 0,
Fixture::kEnableTrueBinary,
static_cast<size_t>(state.range(1) + kEnsureMaxFrameAtLeast),
&stats,
},
b, &outbuf);
if (!logged_representative_output && state.iterations() > 3) {
logged_representative_output = true;
for (size_t i = 0; i < outbuf.count; i++) {
@ -153,7 +147,6 @@ static void BM_HpackEncoderEncodeHeader(benchmark::State& state) {
grpc_core::ExecCtx::Get()->Flush();
}
grpc_metadata_batch_destroy(&b);
grpc_chttp2_hpack_compressor_destroy(c.get());
grpc_slice_buffer_destroy_internal(&outbuf);
std::ostringstream label;

Write Preview
Loading…
Cancel
Save