Add random early rejection for metadata (#32600)

(hopefully last try)

Add new channel arg GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE as hard limit
for metadata. Change GRPC_ARG_MAX_METADATA_SIZE to be a soft limit.
Behavior is as follows:

Hard limit
(1) if hard limit is explicitly set, this will be used.
(2) if hard limit is not explicitly set, maximum of default and soft
limit * 1.25 (if soft limit is set) will be used.

Soft limit
(1) if soft limit is explicitly set, this will be used.
(2) if soft limit is not explicitly set, maximum of default and hard
limit * 0.8 (if hard limit is set) will be used.

Requests between soft and hard limit will be rejected randomly, requests
above hard limit will be rejected.
pull/32650/head
Alisha Nanda 2 years ago committed by GitHub
parent 8d2f70d53c
commit 19d06a78ec
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      BUILD
  2. 3
      CMakeLists.txt
  3. 2
      Makefile
  4. 6
      build_autogenerated.yaml
  5. 1
      config.m4
  6. 1
      config.w32
  7. 2
      gRPC-C++.podspec
  8. 3
      gRPC-Core.podspec
  9. 2
      grpc.gemspec
  10. 2
      grpc.gyp
  11. 13
      include/grpc/impl/grpc_types.h
  12. 2
      package.xml
  13. 1
      src/core/ext/transport/chaotic_good/frame.cc
  14. 35
      src/core/ext/transport/chttp2/transport/chttp2_transport.cc
  15. 65
      src/core/ext/transport/chttp2/transport/hpack_parser.cc
  16. 8
      src/core/ext/transport/chttp2/transport/hpack_parser.h
  17. 2
      src/core/ext/transport/chttp2/transport/internal.h
  18. 6
      src/core/ext/transport/chttp2/transport/parsing.cc
  19. 3
      src/core/lib/backoff/random_early_detection.h
  20. 1
      src/python/grpcio/grpc_core_dependencies.py
  21. 583
      test/core/end2end/tests/large_metadata.cc
  22. 1
      test/core/transport/chttp2/hpack_parser_fuzzer.proto
  23. 10
      test/core/transport/chttp2/hpack_parser_fuzzer_test.cc
  24. 2
      test/core/transport/chttp2/hpack_parser_test.cc
  25. 2
      test/cpp/microbenchmarks/bm_chttp2_hpack.cc
  26. 2
      tools/doxygen/Doxyfile.c++.internal
  27. 2
      tools/doxygen/Doxyfile.core.internal

@ -3572,6 +3572,7 @@ grpc_cc_library(
"//src/core:decode_huff",
"//src/core:error",
"//src/core:hpack_constants",
"//src/core:random_early_detection",
"//src/core:slice",
"//src/core:slice_refcount",
"//src/core:stats_data",

3
CMakeLists.txt generated

@ -2160,6 +2160,7 @@ add_library(grpc
src/core/lib/address_utils/parse_address.cc
src/core/lib/address_utils/sockaddr_utils.cc
src/core/lib/backoff/backoff.cc
src/core/lib/backoff/random_early_detection.cc
src/core/lib/channel/call_tracer.cc
src/core/lib/channel/channel_args.cc
src/core/lib/channel/channel_args_preconditioning.cc
@ -2851,6 +2852,7 @@ add_library(grpc_unsecure
src/core/lib/address_utils/parse_address.cc
src/core/lib/address_utils/sockaddr_utils.cc
src/core/lib/backoff/backoff.cc
src/core/lib/backoff/random_early_detection.cc
src/core/lib/channel/call_tracer.cc
src/core/lib/channel/channel_args.cc
src/core/lib/channel/channel_args_preconditioning.cc
@ -11432,6 +11434,7 @@ add_executable(frame_test
src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c
src/core/lib/address_utils/parse_address.cc
src/core/lib/address_utils/sockaddr_utils.cc
src/core/lib/backoff/random_early_detection.cc
src/core/lib/channel/call_tracer.cc
src/core/lib/channel/channel_args.cc
src/core/lib/channel/channel_args_preconditioning.cc

2
Makefile generated

@ -1404,6 +1404,7 @@ LIBGRPC_SRC = \
src/core/lib/address_utils/parse_address.cc \
src/core/lib/address_utils/sockaddr_utils.cc \
src/core/lib/backoff/backoff.cc \
src/core/lib/backoff/random_early_detection.cc \
src/core/lib/channel/call_tracer.cc \
src/core/lib/channel/channel_args.cc \
src/core/lib/channel/channel_args_preconditioning.cc \
@ -1949,6 +1950,7 @@ LIBGRPC_UNSECURE_SRC = \
src/core/lib/address_utils/parse_address.cc \
src/core/lib/address_utils/sockaddr_utils.cc \
src/core/lib/backoff/backoff.cc \
src/core/lib/backoff/random_early_detection.cc \
src/core/lib/channel/call_tracer.cc \
src/core/lib/channel/channel_args.cc \
src/core/lib/channel/channel_args_preconditioning.cc \

@ -768,6 +768,7 @@ libs:
- src/core/lib/address_utils/sockaddr_utils.h
- src/core/lib/avl/avl.h
- src/core/lib/backoff/backoff.h
- src/core/lib/backoff/random_early_detection.h
- src/core/lib/channel/call_finalization.h
- src/core/lib/channel/call_tracer.h
- src/core/lib/channel/channel_args.h
@ -1557,6 +1558,7 @@ libs:
- src/core/lib/address_utils/parse_address.cc
- src/core/lib/address_utils/sockaddr_utils.cc
- src/core/lib/backoff/backoff.cc
- src/core/lib/backoff/random_early_detection.cc
- src/core/lib/channel/call_tracer.cc
- src/core/lib/channel/channel_args.cc
- src/core/lib/channel/channel_args_preconditioning.cc
@ -2115,6 +2117,7 @@ libs:
- src/core/lib/address_utils/sockaddr_utils.h
- src/core/lib/avl/avl.h
- src/core/lib/backoff/backoff.h
- src/core/lib/backoff/random_early_detection.h
- src/core/lib/channel/call_finalization.h
- src/core/lib/channel/call_tracer.h
- src/core/lib/channel/channel_args.h
@ -2517,6 +2520,7 @@ libs:
- src/core/lib/address_utils/parse_address.cc
- src/core/lib/address_utils/sockaddr_utils.cc
- src/core/lib/backoff/backoff.cc
- src/core/lib/backoff/random_early_detection.cc
- src/core/lib/channel/call_tracer.cc
- src/core/lib/channel/channel_args.cc
- src/core/lib/channel/channel_args_preconditioning.cc
@ -7399,6 +7403,7 @@ targets:
- src/core/lib/address_utils/parse_address.h
- src/core/lib/address_utils/sockaddr_utils.h
- src/core/lib/avl/avl.h
- src/core/lib/backoff/random_early_detection.h
- src/core/lib/channel/call_finalization.h
- src/core/lib/channel/call_tracer.h
- src/core/lib/channel/channel_args.h
@ -7664,6 +7669,7 @@ targets:
- src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c
- src/core/lib/address_utils/parse_address.cc
- src/core/lib/address_utils/sockaddr_utils.cc
- src/core/lib/backoff/random_early_detection.cc
- src/core/lib/channel/call_tracer.cc
- src/core/lib/channel/channel_args.cc
- src/core/lib/channel/channel_args_preconditioning.cc

1
config.m4 generated

@ -485,6 +485,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/lib/address_utils/parse_address.cc \
src/core/lib/address_utils/sockaddr_utils.cc \
src/core/lib/backoff/backoff.cc \
src/core/lib/backoff/random_early_detection.cc \
src/core/lib/channel/call_tracer.cc \
src/core/lib/channel/channel_args.cc \
src/core/lib/channel/channel_args_preconditioning.cc \

1
config.w32 generated

@ -451,6 +451,7 @@ if (PHP_GRPC != "no") {
"src\\core\\lib\\address_utils\\parse_address.cc " +
"src\\core\\lib\\address_utils\\sockaddr_utils.cc " +
"src\\core\\lib\\backoff\\backoff.cc " +
"src\\core\\lib\\backoff\\random_early_detection.cc " +
"src\\core\\lib\\channel\\call_tracer.cc " +
"src\\core\\lib\\channel\\channel_args.cc " +
"src\\core\\lib\\channel\\channel_args_preconditioning.cc " +

2
gRPC-C++.podspec generated

@ -711,6 +711,7 @@ Pod::Spec.new do |s|
'src/core/lib/address_utils/sockaddr_utils.h',
'src/core/lib/avl/avl.h',
'src/core/lib/backoff/backoff.h',
'src/core/lib/backoff/random_early_detection.h',
'src/core/lib/channel/call_finalization.h',
'src/core/lib/channel/call_tracer.h',
'src/core/lib/channel/channel_args.h',
@ -1651,6 +1652,7 @@ Pod::Spec.new do |s|
'src/core/lib/address_utils/sockaddr_utils.h',
'src/core/lib/avl/avl.h',
'src/core/lib/backoff/backoff.h',
'src/core/lib/backoff/random_early_detection.h',
'src/core/lib/channel/call_finalization.h',
'src/core/lib/channel/call_tracer.h',
'src/core/lib/channel/channel_args.h',

3
gRPC-Core.podspec generated

@ -1085,6 +1085,8 @@ Pod::Spec.new do |s|
'src/core/lib/avl/avl.h',
'src/core/lib/backoff/backoff.cc',
'src/core/lib/backoff/backoff.h',
'src/core/lib/backoff/random_early_detection.cc',
'src/core/lib/backoff/random_early_detection.h',
'src/core/lib/channel/call_finalization.h',
'src/core/lib/channel/call_tracer.cc',
'src/core/lib/channel/call_tracer.h',
@ -2340,6 +2342,7 @@ Pod::Spec.new do |s|
'src/core/lib/address_utils/sockaddr_utils.h',
'src/core/lib/avl/avl.h',
'src/core/lib/backoff/backoff.h',
'src/core/lib/backoff/random_early_detection.h',
'src/core/lib/channel/call_finalization.h',
'src/core/lib/channel/call_tracer.h',
'src/core/lib/channel/channel_args.h',

2
grpc.gemspec generated

@ -994,6 +994,8 @@ Gem::Specification.new do |s|
s.files += %w( src/core/lib/avl/avl.h )
s.files += %w( src/core/lib/backoff/backoff.cc )
s.files += %w( src/core/lib/backoff/backoff.h )
s.files += %w( src/core/lib/backoff/random_early_detection.cc )
s.files += %w( src/core/lib/backoff/random_early_detection.h )
s.files += %w( src/core/lib/channel/call_finalization.h )
s.files += %w( src/core/lib/channel/call_tracer.cc )
s.files += %w( src/core/lib/channel/call_tracer.h )

2
grpc.gyp generated

@ -817,6 +817,7 @@
'src/core/lib/address_utils/parse_address.cc',
'src/core/lib/address_utils/sockaddr_utils.cc',
'src/core/lib/backoff/backoff.cc',
'src/core/lib/backoff/random_early_detection.cc',
'src/core/lib/channel/call_tracer.cc',
'src/core/lib/channel/channel_args.cc',
'src/core/lib/channel/channel_args_preconditioning.cc',
@ -1303,6 +1304,7 @@
'src/core/lib/address_utils/parse_address.cc',
'src/core/lib/address_utils/sockaddr_utils.cc',
'src/core/lib/backoff/backoff.cc',
'src/core/lib/backoff/random_early_detection.cc',
'src/core/lib/channel/call_tracer.cc',
'src/core/lib/channel/channel_args.cc',
'src/core/lib/channel/channel_args_preconditioning.cc',

@ -293,9 +293,18 @@ typedef struct {
* protector.
*/
#define GRPC_ARG_TSI_MAX_FRAME_SIZE "grpc.tsi.max_frame_size"
/** Maximum metadata size, in bytes. Note this limit applies to the max sum of
all metadata key-value entries in a batch of headers. */
/** Maximum metadata size (soft limit), in bytes. Note this limit applies to the
max sum of all metadata key-value entries in a batch of headers. Some random
sample of requests between this limit and
`GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE` will be rejected. Defaults to maximum
of 8 KB and `GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE` * 0.8 (if set).
*/
#define GRPC_ARG_MAX_METADATA_SIZE "grpc.max_metadata_size"
/** Maximum metadata size (hard limit), in bytes. Note this limit applies to the
max sum of all metadata key-value entries in a batch of headers. All requests
exceeding this limit will be rejected. Defaults to maximum of 16 KB and
`GRPC_ARG_MAX_METADATA_SIZE` * 1.25 (if set). */
#define GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE "grpc.absolute_max_metadata_size"
/** If non-zero, allow the use of SO_REUSEPORT if it's available (default 1) */
#define GRPC_ARG_ALLOW_REUSEPORT "grpc.so_reuseport"
/** If non-zero, a pointer to a buffer pool (a pointer of type

2
package.xml generated

@ -976,6 +976,8 @@
<file baseinstalldir="/" name="src/core/lib/avl/avl.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/backoff/backoff.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/backoff/backoff.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/backoff/random_early_detection.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/backoff/random_early_detection.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/channel/call_finalization.h" role="src" />
<file baseinstalldir="/" name="src/core/lib/channel/call_tracer.cc" role="src" />
<file baseinstalldir="/" name="src/core/lib/channel/call_tracer.h" role="src" />

@ -156,6 +156,7 @@ absl::StatusOr<Arena::PoolPtr<Metadata>> ReadMetadata(
Arena::PoolPtr<Metadata> metadata;
parser->BeginFrame(
metadata.get(), std::numeric_limits<uint32_t>::max(),
std::numeric_limits<uint32_t>::max(),
is_header ? HPackParser::Boundary::EndOfHeaders
: HPackParser::Boundary::EndOfStream,
HPackParser::Priority::None,

@ -103,7 +103,8 @@
#define DEFAULT_CONNECTION_WINDOW_TARGET (1024 * 1024)
#define MAX_WINDOW 0x7fffffffu
#define MAX_WRITE_BUFFER_SIZE (64 * 1024 * 1024)
#define DEFAULT_MAX_HEADER_LIST_SIZE (8 * 1024)
#define DEFAULT_MAX_HEADER_LIST_SIZE (16 * 1024)
#define DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT (8 * 1024)
#define DEFAULT_CLIENT_KEEPALIVE_TIME_MS INT_MAX
#define DEFAULT_CLIENT_KEEPALIVE_TIMEOUT_MS 20000 // 20 seconds
@ -369,6 +370,21 @@ static void read_channel_args(grpc_chttp2_transport* t,
.GetObjectRef<grpc_core::channelz::SocketNode::Security>());
}
const int soft_limit =
channel_args.GetInt(GRPC_ARG_MAX_METADATA_SIZE).value_or(-1);
if (soft_limit < 0) {
// Set soft limit to 0.8 * hard limit if this is larger than
// `DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT` and
// `GRPC_ARG_MAX_METADATA_SIZE` is not set.
t->max_header_list_size_soft_limit = std::max(
DEFAULT_MAX_HEADER_LIST_SIZE_SOFT_LIMIT,
static_cast<int>(
0.8 * channel_args.GetInt(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE)
.value_or(-1)));
} else {
t->max_header_list_size_soft_limit = soft_limit;
}
static const struct {
absl::string_view channel_arg_name;
grpc_chttp2_setting_id setting_id;
@ -388,7 +404,7 @@ static void read_channel_args(grpc_chttp2_transport* t,
0,
INT32_MAX,
{true, true}},
{GRPC_ARG_MAX_METADATA_SIZE,
{GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE,
GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE,
-1,
0,
@ -421,6 +437,21 @@ static void read_channel_args(grpc_chttp2_transport* t,
if (value >= 0) {
queue_setting_update(t, setting.setting_id,
grpc_core::Clamp(value, setting.min, setting.max));
} else if (setting.setting_id ==
GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE) {
// Set value to 1.25 * soft limit if this is larger than
// `DEFAULT_MAX_HEADER_LIST_SIZE` and
// `GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE` is not set.
const int soft_limit = channel_args.GetInt(GRPC_ARG_MAX_METADATA_SIZE)
.value_or(setting.default_value);
const int value = (soft_limit < (INT_MAX / 1.25))
? static_cast<int>(soft_limit * 1.25)
: soft_limit;
if (value > DEFAULT_MAX_HEADER_LIST_SIZE) {
queue_setting_update(
t, setting.setting_id,
grpc_core::Clamp(value, setting.min, setting.max));
}
}
} else if (channel_args.Contains(setting.channel_arg_name)) {
gpr_log(GPR_DEBUG, "%s is not available on %s",

@ -522,16 +522,17 @@ class HPackParser::String {
// Parser parses one key/value pair from a byte stream.
class HPackParser::Parser {
public:
Parser(Input* input, grpc_metadata_batch* metadata_buffer,
uint32_t metadata_size_limit, HPackTable* table,
Parser(Input* input, grpc_metadata_batch* metadata_buffer, HPackTable* table,
uint8_t* dynamic_table_updates_allowed, uint32_t* frame_length,
RandomEarlyDetection* metadata_early_detection, bool is_last,
LogInfo log_info)
: input_(input),
metadata_buffer_(metadata_buffer),
table_(table),
dynamic_table_updates_allowed_(dynamic_table_updates_allowed),
frame_length_(frame_length),
metadata_size_limit_(metadata_size_limit),
metadata_early_detection_(metadata_early_detection),
is_last_(is_last),
log_info_(log_info) {}
// Skip any priority bits, or return false on failure
@ -655,8 +656,12 @@ class HPackParser::Parser {
// Pass up to the transport
if (GPR_UNLIKELY(metadata_buffer_ == nullptr)) return true;
*frame_length_ += md.transport_size();
if (GPR_UNLIKELY(*frame_length_ > metadata_size_limit_)) {
return HandleMetadataSizeLimitExceeded(md);
if (metadata_early_detection_->MustReject(*frame_length_)) {
// Reject any requests above hard metadata limit.
return HandleMetadataSizeLimitExceeded(md, /*exceeded_hard_limit=*/true);
} else if (is_last_ && metadata_early_detection_->Reject(*frame_length_)) {
// Reject some random sample of requests above soft metadata limit.
return HandleMetadataSizeLimitExceeded(md, /*exceeded_hard_limit=*/false);
}
metadata_buffer_->Set(md);
@ -817,10 +822,12 @@ class HPackParser::Parser {
};
GPR_ATTRIBUTE_NOINLINE
bool HandleMetadataSizeLimitExceeded(const HPackTable::Memento& md) {
bool HandleMetadataSizeLimitExceeded(const HPackTable::Memento& md,
bool exceeded_hard_limit) {
// Collect a summary of sizes so far for debugging
// Do not collect contents, for fear of exposing PII.
std::string summary;
std::string error_message;
if (metadata_buffer_ != nullptr) {
MetadataSizeLimitExceededEncoder encoder(summary);
metadata_buffer_->Encode(&encoder);
@ -828,19 +835,25 @@ class HPackParser::Parser {
summary =
absl::StrCat("; adding ", md.key(), " (length ", md.transport_size(),
"B)", summary.empty() ? "" : " to ", summary);
if (exceeded_hard_limit) {
error_message = absl::StrCat(
"received initial metadata size exceeds hard limit (", *frame_length_,
" vs. ", metadata_early_detection_->hard_limit(), ")", summary);
} else {
error_message = absl::StrCat(
"received initial metadata size exceeds soft limit (", *frame_length_,
" vs. ", metadata_early_detection_->soft_limit(),
"), rejecting requests with some random probability", summary);
}
if (metadata_buffer_ != nullptr) metadata_buffer_->Clear();
// StreamId is used as a signal to skip this stream but keep the connection
// alive
return input_->MaybeSetErrorAndReturn(
[this, summary = std::move(summary)] {
[error_message = std::move(error_message)] {
return grpc_error_set_int(
grpc_error_set_int(
GRPC_ERROR_CREATE(absl::StrCat(
"received initial metadata size exceeds limit (",
*frame_length_, " vs. ", metadata_size_limit_, ")",
summary)),
StatusIntProperty::kRpcStatus,
GRPC_STATUS_RESOURCE_EXHAUSTED),
grpc_error_set_int(GRPC_ERROR_CREATE(error_message),
StatusIntProperty::kRpcStatus,
GRPC_STATUS_RESOURCE_EXHAUSTED),
StatusIntProperty::kStreamId, 0);
},
false);
@ -859,7 +872,9 @@ class HPackParser::Parser {
HPackTable* const table_;
uint8_t* const dynamic_table_updates_allowed_;
uint32_t* const frame_length_;
const uint32_t metadata_size_limit_;
// Random early detection of metadata size limits.
RandomEarlyDetection* metadata_early_detection_;
bool is_last_; // Whether this is the last frame.
const LogInfo log_info_;
};
@ -881,8 +896,10 @@ HPackParser::HPackParser() = default;
HPackParser::~HPackParser() = default;
void HPackParser::BeginFrame(grpc_metadata_batch* metadata_buffer,
uint32_t metadata_size_limit, Boundary boundary,
Priority priority, LogInfo log_info) {
uint32_t metadata_size_soft_limit,
uint32_t metadata_size_hard_limit,
Boundary boundary, Priority priority,
LogInfo log_info) {
metadata_buffer_ = metadata_buffer;
if (metadata_buffer != nullptr) {
metadata_buffer->Set(GrpcStatusFromWire(), true);
@ -891,7 +908,9 @@ void HPackParser::BeginFrame(grpc_metadata_batch* metadata_buffer,
priority_ = priority;
dynamic_table_updates_allowed_ = 2;
frame_length_ = 0;
metadata_size_limit_ = metadata_size_limit;
metadata_early_detection_ = RandomEarlyDetection(
/*soft_limit=*/metadata_size_soft_limit,
/*hard_limit=*/metadata_size_hard_limit);
log_info_ = log_info;
}
@ -909,7 +928,7 @@ grpc_error_handle HPackParser::Parse(const grpc_slice& slice, bool is_last) {
}
grpc_error_handle HPackParser::ParseInput(Input input, bool is_last) {
bool parsed_ok = ParseInputInner(&input);
bool parsed_ok = ParseInputInner(&input, is_last);
if (is_last) global_stats().IncrementHttp2MetadataSize(frame_length_);
if (parsed_ok) return absl::OkStatus();
if (input.eof_error()) {
@ -923,7 +942,7 @@ grpc_error_handle HPackParser::ParseInput(Input input, bool is_last) {
return input.TakeError();
}
bool HPackParser::ParseInputInner(Input* input) {
bool HPackParser::ParseInputInner(Input* input, bool is_last) {
switch (priority_) {
case Priority::None:
break;
@ -935,9 +954,9 @@ bool HPackParser::ParseInputInner(Input* input) {
}
}
while (!input->end_of_stream()) {
if (GPR_UNLIKELY(!Parser(input, metadata_buffer_, metadata_size_limit_,
&table_, &dynamic_table_updates_allowed_,
&frame_length_, log_info_)
if (GPR_UNLIKELY(!Parser(input, metadata_buffer_, &table_,
&dynamic_table_updates_allowed_, &frame_length_,
&metadata_early_detection_, is_last, log_info_)
.Parse())) {
return false;
}

@ -29,6 +29,7 @@
#include "src/core/ext/transport/chttp2/transport/frame.h"
#include "src/core/ext/transport/chttp2/transport/hpack_parser_table.h"
#include "src/core/lib/backoff/random_early_detection.h"
#include "src/core/lib/iomgr/error.h"
#include "src/core/lib/transport/metadata_batch.h"
@ -80,7 +81,8 @@ class HPackParser {
// Begin parsing a new frame
// Sink receives each parsed header,
void BeginFrame(grpc_metadata_batch* metadata_buffer,
uint32_t metadata_size_limit, Boundary boundary,
uint32_t metadata_size_soft_limit,
uint32_t metadata_size_hard_limit, Boundary boundary,
Priority priority, LogInfo log_info);
// Start throwing away any received headers after parsing them.
void StopBufferingFrame() { metadata_buffer_ = nullptr; }
@ -103,7 +105,7 @@ class HPackParser {
class String;
grpc_error_handle ParseInput(Input input, bool is_last);
bool ParseInputInner(Input* input);
bool ParseInputInner(Input* input, bool is_last);
// Target metadata buffer
grpc_metadata_batch* metadata_buffer_ = nullptr;
@ -121,7 +123,7 @@ class HPackParser {
uint8_t dynamic_table_updates_allowed_;
// Length of frame so far.
uint32_t frame_length_;
uint32_t metadata_size_limit_;
RandomEarlyDetection metadata_early_detection_;
// Information for logging
LogInfo log_info_;

@ -465,6 +465,8 @@ struct grpc_chttp2_transport
bool keepalive_ping_started = false;
/// keep-alive state machine state
grpc_chttp2_keepalive_state keepalive_state;
// Soft limit on max header size.
uint32_t max_header_list_size_soft_limit = 0;
grpc_core::ContextList* cl = nullptr;
grpc_core::RefCountedPtr<grpc_core::channelz::SocketNode> channelz_socket;
uint32_t num_messages_in_next_write = 0;

@ -476,6 +476,9 @@ static grpc_error_handle init_header_skip_frame_parser(
"header", grpc_chttp2_header_parser_parse, &t->hpack_parser};
t->hpack_parser.BeginFrame(
nullptr,
/*metadata_size_soft_limit=*/
t->max_header_list_size_soft_limit,
/*metadata_size_hard_limit=*/
t->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE],
hpack_boundary_type(t, is_eoh), priority_type,
@ -692,6 +695,9 @@ static grpc_error_handle init_header_frame_parser(grpc_chttp2_transport* t,
}
t->hpack_parser.BeginFrame(
incoming_metadata_buffer,
/*metadata_size_soft_limit=*/
t->max_header_list_size_soft_limit,
/*metadata_size_hard_limit=*/
t->settings[GRPC_ACKED_SETTINGS]
[GRPC_CHTTP2_SETTINGS_MAX_HEADER_LIST_SIZE],
hpack_boundary_type(t, is_eoh), priority_type,

@ -17,6 +17,8 @@
#include <grpc/support/port_platform.h>
#include <limits.h>
#include <cstdint>
#include "absl/random/random.h"
@ -27,6 +29,7 @@ namespace grpc_core {
// or accepted based upon their size.
class RandomEarlyDetection {
public:
RandomEarlyDetection() : soft_limit_(INT_MAX), hard_limit_(INT_MAX) {}
RandomEarlyDetection(uint64_t soft_limit, uint64_t hard_limit)
: soft_limit_(soft_limit), hard_limit_(hard_limit) {}

@ -460,6 +460,7 @@ CORE_SOURCE_FILES = [
'src/core/lib/address_utils/parse_address.cc',
'src/core/lib/address_utils/sockaddr_utils.cc',
'src/core/lib/backoff/backoff.cc',
'src/core/lib/backoff/random_early_detection.cc',
'src/core/lib/channel/call_tracer.cc',
'src/core/lib/channel/channel_args.cc',
'src/core/lib/channel/channel_args_preconditioning.cc',

@ -17,12 +17,12 @@
//
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <functional>
#include <memory>
#include <grpc/byte_buffer.h>
#include <grpc/grpc.h>
#include <grpc/impl/propagation_bits.h>
#include <grpc/slice.h>
@ -31,6 +31,7 @@
#include <grpc/support/time.h>
#include "src/core/lib/channel/channel_args.h"
#include "src/core/lib/gpr/useful.h"
#include "test/core/end2end/cq_verifier.h"
#include "test/core/end2end/end2end_tests.h"
#include "test/core/util/test_config.h"
@ -46,45 +47,32 @@ static std::unique_ptr<CoreTestFixture> begin_test(
return f;
}
// Request with a large amount of metadata.
static void test_request_with_large_metadata(
const CoreTestConfiguration& config) {
static grpc_status_code send_metadata(CoreTestFixture* f,
const size_t metadata_size,
grpc_slice* client_details) {
grpc_core::CqVerifier cqv(f->cq());
grpc_call* c;
grpc_call* s;
grpc_slice request_payload_slice =
grpc_slice_from_copied_string("hello world");
grpc_byte_buffer* request_payload =
grpc_raw_byte_buffer_create(&request_payload_slice, 1);
grpc_metadata meta;
const size_t large_size = 64 * 1024;
grpc_arg arg;
arg.type = GRPC_ARG_INTEGER;
arg.key = const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE);
arg.value.integer = static_cast<int>(large_size) + 1024;
grpc_channel_args args = {1, &arg};
auto f = begin_test(config, "test_request_with_large_metadata", &args, &args);
grpc_core::CqVerifier cqv(f->cq());
grpc_op ops[6];
grpc_op* op;
grpc_metadata_array initial_metadata_recv;
grpc_metadata_array trailing_metadata_recv;
grpc_metadata_array request_metadata_recv;
grpc_byte_buffer* request_payload_recv = nullptr;
grpc_call_details call_details;
grpc_status_code status;
grpc_call_error error;
grpc_slice details;
int was_cancelled = 2;
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5);
c = grpc_channel_create_call(f->client(), nullptr, GRPC_PROPAGATE_DEFAULTS,
f->cq(), grpc_slice_from_static_string("/foo"),
nullptr, deadline, nullptr);
GPR_ASSERT(c);
// Add metadata of size `metadata_size`.
meta.key = grpc_slice_from_static_string("key");
meta.value = grpc_slice_malloc(large_size);
memset(GRPC_SLICE_START_PTR(meta.value), 'a', large_size);
meta.value = grpc_slice_malloc(metadata_size);
memset(GRPC_SLICE_START_PTR(meta.value), 'a', metadata_size);
grpc_metadata_array_init(&initial_metadata_recv);
grpc_metadata_array_init(&trailing_metadata_recv);
@ -92,16 +80,10 @@ static void test_request_with_large_metadata(
grpc_call_details_init(&call_details);
memset(ops, 0, sizeof(ops));
// Client: send request.
// Client: wait on initial metadata from server.
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 1;
op->data.send_initial_metadata.metadata = &meta;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_MESSAGE;
op->data.send_message.send_message = request_payload;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
@ -117,7 +99,7 @@ static void test_request_with_large_metadata(
op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
op->data.recv_status_on_client.status = &status;
op->data.recv_status_on_client.status_details = &details;
op->data.recv_status_on_client.status_details = client_details;
op->flags = 0;
op->reserved = nullptr;
op++;
@ -134,29 +116,14 @@ static void test_request_with_large_metadata(
cqv.Verify();
memset(ops, 0, sizeof(ops));
// Server: send initial metadata and receive request.
// Server: send metadata of size `metadata_size`.
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_MESSAGE;
op->data.recv_message.recv_message = &request_payload_recv;
op->data.send_initial_metadata.count = 1;
op->data.send_initial_metadata.metadata = &meta;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops),
grpc_core::CqVerifier::tag(102), nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
cqv.Expect(grpc_core::CqVerifier::tag(102), true);
cqv.Verify();
memset(ops, 0, sizeof(ops));
// Server: receive close and send status. This should trigger
// completion of request on client.
op = ops;
op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
op->data.recv_close_on_server.cancelled = &was_cancelled;
op->flags = 0;
@ -171,23 +138,12 @@ static void test_request_with_large_metadata(
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops),
grpc_core::CqVerifier::tag(103), nullptr);
grpc_core::CqVerifier::tag(102), nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
cqv.Expect(grpc_core::CqVerifier::tag(103), true);
cqv.Expect(grpc_core::CqVerifier::tag(102), true);
cqv.Expect(grpc_core::CqVerifier::tag(1), true);
cqv.Verify();
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));
GPR_ASSERT(was_cancelled == 0);
GPR_ASSERT(byte_buffer_eq_string(request_payload_recv, "hello world"));
GPR_ASSERT(contains_metadata_slices(&request_metadata_recv,
grpc_slice_from_static_string("key"),
meta.value));
grpc_slice_unref(details);
grpc_metadata_array_destroy(&initial_metadata_recv);
grpc_metadata_array_destroy(&trailing_metadata_recv);
grpc_metadata_array_destroy(&request_metadata_recv);
@ -196,147 +152,408 @@ static void test_request_with_large_metadata(
grpc_call_unref(c);
grpc_call_unref(s);
grpc_byte_buffer_destroy(request_payload);
grpc_byte_buffer_destroy(request_payload_recv);
grpc_slice_unref(meta.value);
return status;
}
// Server responds with metadata larger than what the client accepts.
static void test_request_with_bad_large_metadata_response(
// Server responds with metadata under soft limit of what client accepts. No
// requests should be rejected.
static void test_request_with_large_metadata_under_soft_limit(
const CoreTestConfiguration& config) {
grpc_arg arg;
arg.type = GRPC_ARG_INTEGER;
arg.key = const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE);
arg.value.integer = 1024;
grpc_channel_args args = {1, &arg};
auto f = begin_test(config, "test_request_with_bad_large_metadata_response",
&args, &args);
grpc_core::CqVerifier cqv(f->cq());
const size_t soft_limit = 32 * 1024;
const size_t hard_limit = 45 * 1024;
const size_t metadata_size = soft_limit;
grpc_arg arg[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE), soft_limit + 1024),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE),
hard_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f =
begin_test(config, "test_request_with_large_metadata_under_soft_limit",
&args, &args);
for (int i = 0; i < 100; i++) {
grpc_slice client_details;
auto status = send_metadata(f.get(), metadata_size, &client_details);
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
grpc_slice_unref(client_details);
}
}
for (int i = 0; i < 10; i++) {
grpc_call* c;
grpc_call* s;
grpc_metadata meta;
const size_t large_size = 64 * 1024;
grpc_op ops[6];
grpc_op* op;
grpc_metadata_array initial_metadata_recv;
grpc_metadata_array trailing_metadata_recv;
grpc_metadata_array request_metadata_recv;
grpc_call_details call_details;
grpc_status_code status;
grpc_call_error error;
grpc_slice details;
int was_cancelled = 2;
gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5);
c = grpc_channel_create_call(f->client(), nullptr, GRPC_PROPAGATE_DEFAULTS,
f->cq(), grpc_slice_from_static_string("/foo"),
nullptr, deadline, nullptr);
GPR_ASSERT(c);
meta.key = grpc_slice_from_static_string("key");
meta.value = grpc_slice_malloc(large_size);
memset(GRPC_SLICE_START_PTR(meta.value), 'a', large_size);
grpc_metadata_array_init(&initial_metadata_recv);
grpc_metadata_array_init(&trailing_metadata_recv);
grpc_metadata_array_init(&request_metadata_recv);
grpc_call_details_init(&call_details);
memset(ops, 0, sizeof(ops));
// Client: send request.
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 0;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_INITIAL_METADATA;
op->data.recv_initial_metadata.recv_initial_metadata =
&initial_metadata_recv;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
op->data.recv_status_on_client.status = &status;
op->data.recv_status_on_client.status_details = &details;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops),
grpc_core::CqVerifier::tag(1), nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
error = grpc_server_request_call(f->server(), &s, &call_details,
&request_metadata_recv, f->cq(), f->cq(),
grpc_core::CqVerifier::tag(101));
GPR_ASSERT(GRPC_CALL_OK == error);
cqv.Expect(grpc_core::CqVerifier::tag(101), true);
cqv.Verify();
memset(ops, 0, sizeof(ops));
// Server: send large initial metadata
op = ops;
op->op = GRPC_OP_SEND_INITIAL_METADATA;
op->data.send_initial_metadata.count = 1;
op->data.send_initial_metadata.metadata = &meta;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
op->data.recv_close_on_server.cancelled = &was_cancelled;
op->flags = 0;
op->reserved = nullptr;
op++;
op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
op->data.send_status_from_server.trailing_metadata_count = 0;
op->data.send_status_from_server.status = GRPC_STATUS_OK;
grpc_slice status_details = grpc_slice_from_static_string("xyz");
op->data.send_status_from_server.status_details = &status_details;
op->flags = 0;
op->reserved = nullptr;
op++;
error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops),
grpc_core::CqVerifier::tag(102), nullptr);
GPR_ASSERT(GRPC_CALL_OK == error);
cqv.Expect(grpc_core::CqVerifier::tag(102), true);
cqv.Expect(grpc_core::CqVerifier::tag(1), true);
cqv.Verify();
// Server responds with metadata between soft and hard limits of what client
// accepts. Some requests should be rejected.
static void test_request_with_large_metadata_between_soft_and_hard_limits(
const CoreTestConfiguration& config) {
const size_t soft_limit = 32 * 1024;
const size_t hard_limit = 45 * 1024;
const size_t metadata_size = (soft_limit + hard_limit) / 2;
grpc_arg arg[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE), soft_limit + 1024),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE),
hard_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f = begin_test(
config, "test_request_with_large_metadata_between_soft_and_hard_limits",
&args, &args);
int num_requests_rejected = 0;
for (int i = 0; i < 100; i++) {
grpc_slice client_details;
auto status = send_metadata(f.get(), metadata_size, &client_details);
if (status == GRPC_STATUS_RESOURCE_EXHAUSTED) {
num_requests_rejected++;
const char* expected_error =
"received initial metadata size exceeds soft limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
} else {
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
}
grpc_slice_unref(client_details);
}
// Check that some requests were rejected.
GPR_ASSERT(abs(num_requests_rejected - 50) <= 45);
}
// Server responds with metadata above hard limit of what the client accepts.
// All requests should be rejected.
static void test_request_with_large_metadata_above_hard_limit(
const CoreTestConfiguration& config) {
const size_t soft_limit = 32 * 1024;
const size_t hard_limit = 45 * 1024;
const size_t metadata_size = hard_limit * 1.5;
grpc_arg arg[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE), soft_limit + 1024),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE),
hard_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f =
begin_test(config, "test_request_with_large_metadata_above_hard_limit",
&args, &args);
for (int i = 0; i < 100; i++) {
grpc_slice client_details;
auto status = send_metadata(f.get(), metadata_size, &client_details);
GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED);
const char* expected_error = "received initial metadata size exceeds limit";
const char* expected_error =
"received initial metadata size exceeds hard limit";
grpc_slice actual_error =
grpc_slice_split_head(&details, strlen(expected_error));
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));
grpc_slice_unref(actual_error);
grpc_slice_unref(client_details);
}
}
// Set soft limit higher than hard limit. All requests above hard limit should
// be rejected, all requests below hard limit should be accepted (soft limit
// should not be respected).
static void test_request_with_large_metadata_soft_limit_above_hard_limit(
const CoreTestConfiguration& config) {
const size_t soft_limit = 64 * 1024;
const size_t hard_limit = 32 * 1024;
const size_t metadata_size_below_hard_limit = hard_limit;
const size_t metadata_size_above_hard_limit = hard_limit * 2;
grpc_arg arg[] = {
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE), soft_limit + 1024),
grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE),
hard_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f = begin_test(
config, "test_request_with_large_metadata_soft_limit_above_hard_limit",
&args, &args);
// Send 50 requests below hard limit. Should be accepted.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_below_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
grpc_slice_unref(client_details);
}
// Send 50 requests above hard limit. Should be rejected.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_above_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED);
const char* expected_error =
"received initial metadata size exceeds hard limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
grpc_slice_unref(client_details);
}
}
// Set soft limit * 1.25 higher than default hard limit and do not set hard
// limit. Soft limit * 1.25 should be used as hard limit.
static void test_request_with_large_metadata_soft_limit_overrides_default_hard(
const CoreTestConfiguration& config) {
const size_t soft_limit = 64 * 1024;
const size_t metadata_size_below_soft_limit = soft_limit;
const size_t metadata_size_above_hard_limit = soft_limit * 1.5;
const size_t metadata_size_between_limits =
(soft_limit + soft_limit * 1.25) / 2;
grpc_arg arg[] = {grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE), soft_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f = begin_test(
config,
"test_request_with_large_metadata_soft_limit_overrides_default_hard",
&args, &args);
// Send 50 requests below soft limit. Should be accepted.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_below_soft_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
grpc_slice_unref(client_details);
}
// Send 100 requests between soft and hard limits. Some should be rejected.
int num_requests_rejected = 0;
for (int i = 0; i < 100; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_between_limits, &client_details);
if (status == GRPC_STATUS_RESOURCE_EXHAUSTED) {
num_requests_rejected++;
const char* expected_error =
"received initial metadata size exceeds soft limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
} else {
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
}
grpc_slice_unref(client_details);
}
// Check that some requests were rejected.
GPR_ASSERT(abs(num_requests_rejected - 50) <= 45);
// Send 50 requests above hard limit. Should be rejected.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_above_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED);
const char* expected_error =
"received initial metadata size exceeds hard limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
grpc_slice_unref(client_details);
}
}
// Set hard limit * 0.8 higher than default soft limit and do not set soft
// limit. Hard limit * 0.8 should be used as soft limit.
static void test_request_with_large_metadata_hard_limit_overrides_default_soft(
const CoreTestConfiguration& config) {
const size_t hard_limit = 45 * 1024;
const size_t metadata_size_below_soft_limit = hard_limit * 0.5;
const size_t metadata_size_above_hard_limit = hard_limit * 1.5;
const size_t metadata_size_between_limits =
(hard_limit * 0.8 + hard_limit) / 2;
grpc_arg arg[] = {grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE),
hard_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f = begin_test(
config,
"test_request_with_large_metadata_hard_limit_overrides_default_soft",
&args, &args);
// Send 50 requests below soft limit. Should be accepted.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_below_soft_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
grpc_slice_unref(client_details);
}
// Send 100 requests between soft and hard limits. Some should be rejected.
int num_requests_rejected = 0;
for (int i = 0; i < 100; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_between_limits, &client_details);
if (status == GRPC_STATUS_RESOURCE_EXHAUSTED) {
num_requests_rejected++;
const char* expected_error =
"received initial metadata size exceeds soft limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
} else {
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
}
grpc_slice_unref(client_details);
}
// Check that some requests were rejected.
GPR_ASSERT(abs(num_requests_rejected - 50) <= 45);
// Send 50 requests above hard limit. Should be rejected.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_above_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED);
const char* expected_error =
"received initial metadata size exceeds hard limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
grpc_slice_unref(client_details);
}
}
// Set hard limit lower than default hard limit and ensure new limit is
// respected. Default soft limit is not respected since hard limit is lower than
// soft limit.
static void test_request_with_large_metadata_hard_limit_below_default_hard(
const CoreTestConfiguration& config) {
const size_t hard_limit = 4 * 1024;
const size_t metadata_size_below_hard_limit = hard_limit;
const size_t metadata_size_above_hard_limit = hard_limit * 2;
grpc_arg arg[] = {grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_ABSOLUTE_MAX_METADATA_SIZE),
hard_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f = begin_test(
config, "test_request_with_large_metadata_hard_limit_below_default_hard",
&args, &args);
// Send 50 requests below hard limit. Should be accepted.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_below_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
grpc_slice_unref(client_details);
}
// Send 50 requests above hard limit. Should be rejected.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_above_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED);
const char* expected_error =
"received initial metadata size exceeds hard limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
grpc_slice_unref(details);
grpc_metadata_array_destroy(&initial_metadata_recv);
grpc_metadata_array_destroy(&trailing_metadata_recv);
grpc_metadata_array_destroy(&request_metadata_recv);
grpc_call_details_destroy(&call_details);
grpc_slice_unref(client_details);
}
}
grpc_call_unref(c);
grpc_call_unref(s);
// Set soft limit lower than default soft limit and ensure new limit is
// respected. Hard limit should be default hard since this is greater than 2 *
// soft limit.
static void test_request_with_large_metadata_soft_limit_below_default_soft(
const CoreTestConfiguration& config) {
const size_t soft_limit = 1 * 1024;
const size_t metadata_size_below_soft_limit = soft_limit;
// greater than 2 * soft, less than default hard
const size_t metadata_size_between_limits = 10 * 1024;
const size_t metadata_size_above_hard_limit = 75 * 1024;
grpc_arg arg[] = {grpc_channel_arg_integer_create(
const_cast<char*>(GRPC_ARG_MAX_METADATA_SIZE), soft_limit + 1024)};
grpc_channel_args args = {GPR_ARRAY_SIZE(arg), arg};
auto f = begin_test(
config, "test_request_with_large_metadata_soft_limit_below_default_soft",
&args, &args);
// Send 50 requests below soft limit. Should be accepted.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_below_soft_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
grpc_slice_unref(client_details);
}
grpc_slice_unref(meta.value);
// Send 100 requests between soft and hard limits. Some should be rejected.
int num_requests_rejected = 0;
for (int i = 0; i < 100; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_between_limits, &client_details);
if (status == GRPC_STATUS_RESOURCE_EXHAUSTED) {
num_requests_rejected++;
const char* expected_error =
"received initial metadata size exceeds soft limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
} else {
GPR_ASSERT(status == GRPC_STATUS_OK);
GPR_ASSERT(0 == grpc_slice_str_cmp(client_details, "xyz"));
}
grpc_slice_unref(client_details);
}
// Check that some requests were rejected.
GPR_ASSERT((abs(num_requests_rejected - 50) <= 49));
// Send 50 requests above hard limit. Should be rejected.
for (int i = 0; i < 50; i++) {
grpc_slice client_details;
auto status =
send_metadata(f.get(), metadata_size_above_hard_limit, &client_details);
GPR_ASSERT(status == GRPC_STATUS_RESOURCE_EXHAUSTED);
const char* expected_error =
"received initial metadata size exceeds hard limit";
grpc_slice actual_error =
grpc_slice_split_head(&client_details, strlen(expected_error));
GPR_ASSERT(0 == grpc_slice_str_cmp(actual_error, expected_error));
grpc_slice_unref(actual_error);
grpc_slice_unref(client_details);
}
}
void large_metadata(const CoreTestConfiguration& config) {
test_request_with_large_metadata(config);
test_request_with_large_metadata_under_soft_limit(config);
// TODO(yashykt): Maybe add checks for metadata size in inproc transport too.
if (strcmp(config.name, "inproc") != 0) {
test_request_with_bad_large_metadata_response(config);
test_request_with_large_metadata_between_soft_and_hard_limits(config);
test_request_with_large_metadata_above_hard_limit(config);
test_request_with_large_metadata_soft_limit_above_hard_limit(config);
test_request_with_large_metadata_soft_limit_overrides_default_hard(config);
test_request_with_large_metadata_hard_limit_overrides_default_soft(config);
test_request_with_large_metadata_hard_limit_below_default_hard(config);
test_request_with_large_metadata_soft_limit_below_default_soft(config);
}
}

@ -25,6 +25,7 @@ message Frame {
int32 stop_buffering_after_segments = 4;
int32 max_metadata_length = 5;
repeated bytes parse = 6;
int32 absolute_max_metadata_length = 7;
}
message Msg {

@ -19,6 +19,7 @@
#include <algorithm>
#include <memory>
#include <string>
#include <utility>
#include <grpc/grpc.h>
#include <grpc/slice.h>
@ -69,12 +70,19 @@ DEFINE_PROTO_FUZZER(const hpack_parser_fuzzer::Msg& msg) {
priority = grpc_core::HPackParser::Priority::Included;
}
int max_length = 1024;
int absolute_max_length = 1024;
if (absolute_max_length < max_length) {
std::swap(absolute_max_length, max_length);
}
if (frame.max_metadata_length() != 0) {
max_length = frame.max_metadata_length();
}
if (frame.absolute_max_metadata_length() != 0) {
absolute_max_length = frame.absolute_max_metadata_length();
}
parser->BeginFrame(
&b, max_length, boundary, priority,
&b, max_length, absolute_max_length, boundary, priority,
grpc_core::HPackParser::LogInfo{
1, grpc_core::HPackParser::LogInfo::kHeaders, false});
int stop_buffering_ctr =

@ -98,7 +98,7 @@ class ParseTest : public ::testing::TestWithParam<Test> {
grpc_metadata_batch b(arena.get());
parser_->BeginFrame(
&b, 4096, grpc_core::HPackParser::Boundary::None,
&b, 4096, 4096, grpc_core::HPackParser::Boundary::None,
grpc_core::HPackParser::Priority::None,
grpc_core::HPackParser::LogInfo{
1, grpc_core::HPackParser::LogInfo::kHeaders, false});

@ -349,6 +349,7 @@ static void BM_HpackParserParseHeader(benchmark::State& state) {
grpc_core::ManualConstructor<grpc_metadata_batch> b;
b.Init(arena);
p.BeginFrame(&*b, std::numeric_limits<uint32_t>::max(),
std::numeric_limits<uint32_t>::max(),
grpc_core::HPackParser::Boundary::None,
grpc_core::HPackParser::Priority::None,
grpc_core::HPackParser::LogInfo{
@ -371,6 +372,7 @@ static void BM_HpackParserParseHeader(benchmark::State& state) {
arena = grpc_core::Arena::Create(kArenaSize, &memory_allocator);
b.Init(arena);
p.BeginFrame(&*b, std::numeric_limits<uint32_t>::max(),
std::numeric_limits<uint32_t>::max(),
grpc_core::HPackParser::Boundary::None,
grpc_core::HPackParser::Priority::None,
grpc_core::HPackParser::LogInfo{

@ -1989,6 +1989,8 @@ src/core/lib/address_utils/sockaddr_utils.h \
src/core/lib/avl/avl.h \
src/core/lib/backoff/backoff.cc \
src/core/lib/backoff/backoff.h \
src/core/lib/backoff/random_early_detection.cc \
src/core/lib/backoff/random_early_detection.h \
src/core/lib/channel/call_finalization.h \
src/core/lib/channel/call_tracer.cc \
src/core/lib/channel/call_tracer.h \

@ -1766,6 +1766,8 @@ src/core/lib/address_utils/sockaddr_utils.h \
src/core/lib/avl/avl.h \
src/core/lib/backoff/backoff.cc \
src/core/lib/backoff/backoff.h \
src/core/lib/backoff/random_early_detection.cc \
src/core/lib/backoff/random_early_detection.h \
src/core/lib/channel/README.md \
src/core/lib/channel/call_finalization.h \
src/core/lib/channel/call_tracer.cc \

Loading…
Cancel
Save