Merge pull request #16488 from hcaseyal/hpack_static_table

Hpack changes part 1: make grpc use the static hpack table on the encode path and optimize encoding
pull/13653/merge
hcaseyal 6 years ago committed by GitHub
commit 5bb69e0f01
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      BUILD
  2. 6
      CMakeLists.txt
  3. 6
      Makefile
  4. 2
      build.yaml
  5. 1
      config.m4
  6. 1
      config.w32
  7. 1
      gRPC-C++.podspec
  8. 3
      gRPC-Core.podspec
  9. 2
      grpc.gemspec
  10. 4
      grpc.gyp
  11. 2
      package.xml
  12. 15
      src/core/ext/transport/chttp2/transport/hpack_encoder.cc
  13. 39
      src/core/ext/transport/chttp2/transport/hpack_mapping.cc
  14. 38
      src/core/ext/transport/chttp2/transport/hpack_mapping.h
  15. 29
      src/core/ext/transport/chttp2/transport/hpack_table.cc
  16. 9
      src/core/ext/transport/chttp2/transport/hpack_table.h
  17. 18
      src/core/lib/transport/metadata.cc
  18. 3
      src/core/lib/transport/metadata.h
  19. 2
      src/core/lib/transport/metadata_batch.h
  20. 1
      src/python/grpcio/grpc_core_dependencies.py
  21. 2
      test/core/transport/chttp2/hpack_encoder_test.cc
  22. 5
      test/core/transport/metadata_test.cc
  23. 208
      tools/codegen/core/gen_static_metadata.py
  24. 2
      tools/doxygen/Doxyfile.core.internal
  25. 3
      tools/run_tests/generated/sources_and_headers.json

@ -1575,6 +1575,7 @@ grpc_cc_library(
"src/core/ext/transport/chttp2/transport/frame_window_update.cc",
"src/core/ext/transport/chttp2/transport/hpack_encoder.cc",
"src/core/ext/transport/chttp2/transport/hpack_parser.cc",
"src/core/ext/transport/chttp2/transport/hpack_mapping.cc",
"src/core/ext/transport/chttp2/transport/hpack_table.cc",
"src/core/ext/transport/chttp2/transport/http2_settings.cc",
"src/core/ext/transport/chttp2/transport/huffsyms.cc",
@ -1599,6 +1600,7 @@ grpc_cc_library(
"src/core/ext/transport/chttp2/transport/frame_window_update.h",
"src/core/ext/transport/chttp2/transport/hpack_encoder.h",
"src/core/ext/transport/chttp2/transport/hpack_parser.h",
"src/core/ext/transport/chttp2/transport/hpack_mapping.h",
"src/core/ext/transport/chttp2/transport/hpack_table.h",
"src/core/ext/transport/chttp2/transport/http2_settings.h",
"src/core/ext/transport/chttp2/transport/huffsyms.h",

@ -1105,6 +1105,7 @@ add_library(grpc
src/core/ext/transport/chttp2/transport/frame_settings.cc
src/core/ext/transport/chttp2/transport/frame_window_update.cc
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
src/core/ext/transport/chttp2/transport/hpack_mapping.cc
src/core/ext/transport/chttp2/transport/hpack_parser.cc
src/core/ext/transport/chttp2/transport/hpack_table.cc
src/core/ext/transport/chttp2/transport/http2_settings.cc
@ -1516,6 +1517,7 @@ add_library(grpc_cronet
src/core/ext/transport/chttp2/transport/frame_settings.cc
src/core/ext/transport/chttp2/transport/frame_window_update.cc
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
src/core/ext/transport/chttp2/transport/hpack_mapping.cc
src/core/ext/transport/chttp2/transport/hpack_parser.cc
src/core/ext/transport/chttp2/transport/hpack_table.cc
src/core/ext/transport/chttp2/transport/http2_settings.cc
@ -1929,6 +1931,7 @@ add_library(grpc_test_util
src/core/ext/transport/chttp2/transport/frame_settings.cc
src/core/ext/transport/chttp2/transport/frame_window_update.cc
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
src/core/ext/transport/chttp2/transport/hpack_mapping.cc
src/core/ext/transport/chttp2/transport/hpack_parser.cc
src/core/ext/transport/chttp2/transport/hpack_table.cc
src/core/ext/transport/chttp2/transport/http2_settings.cc
@ -2239,6 +2242,7 @@ add_library(grpc_test_util_unsecure
src/core/ext/transport/chttp2/transport/frame_settings.cc
src/core/ext/transport/chttp2/transport/frame_window_update.cc
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
src/core/ext/transport/chttp2/transport/hpack_mapping.cc
src/core/ext/transport/chttp2/transport/hpack_parser.cc
src/core/ext/transport/chttp2/transport/hpack_table.cc
src/core/ext/transport/chttp2/transport/http2_settings.cc
@ -2507,6 +2511,7 @@ add_library(grpc_unsecure
src/core/ext/transport/chttp2/transport/frame_settings.cc
src/core/ext/transport/chttp2/transport/frame_window_update.cc
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
src/core/ext/transport/chttp2/transport/hpack_mapping.cc
src/core/ext/transport/chttp2/transport/hpack_parser.cc
src/core/ext/transport/chttp2/transport/hpack_table.cc
src/core/ext/transport/chttp2/transport/http2_settings.cc
@ -3178,6 +3183,7 @@ add_library(grpc++_cronet
src/core/ext/transport/chttp2/transport/frame_settings.cc
src/core/ext/transport/chttp2/transport/frame_window_update.cc
src/core/ext/transport/chttp2/transport/hpack_encoder.cc
src/core/ext/transport/chttp2/transport/hpack_mapping.cc
src/core/ext/transport/chttp2/transport/hpack_parser.cc
src/core/ext/transport/chttp2/transport/hpack_table.cc
src/core/ext/transport/chttp2/transport/http2_settings.cc

@ -3614,6 +3614,7 @@ LIBGRPC_SRC = \
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \
@ -4024,6 +4025,7 @@ LIBGRPC_CRONET_SRC = \
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \
@ -4435,6 +4437,7 @@ LIBGRPC_TEST_UTIL_SRC = \
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \
@ -4736,6 +4739,7 @@ LIBGRPC_TEST_UTIL_UNSECURE_SRC = \
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \
@ -4982,6 +4986,7 @@ LIBGRPC_UNSECURE_SRC = \
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \
@ -5641,6 +5646,7 @@ LIBGRPC++_CRONET_SRC = \
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \

@ -936,6 +936,7 @@ filegroups:
- src/core/ext/transport/chttp2/transport/frame_settings.h
- src/core/ext/transport/chttp2/transport/frame_window_update.h
- src/core/ext/transport/chttp2/transport/hpack_encoder.h
- src/core/ext/transport/chttp2/transport/hpack_mapping.h
- src/core/ext/transport/chttp2/transport/hpack_parser.h
- src/core/ext/transport/chttp2/transport/hpack_table.h
- src/core/ext/transport/chttp2/transport/http2_settings.h
@ -957,6 +958,7 @@ filegroups:
- src/core/ext/transport/chttp2/transport/frame_settings.cc
- src/core/ext/transport/chttp2/transport/frame_window_update.cc
- src/core/ext/transport/chttp2/transport/hpack_encoder.cc
- src/core/ext/transport/chttp2/transport/hpack_mapping.cc
- src/core/ext/transport/chttp2/transport/hpack_parser.cc
- src/core/ext/transport/chttp2/transport/hpack_table.cc
- src/core/ext/transport/chttp2/transport/http2_settings.cc

@ -249,6 +249,7 @@ if test "$PHP_GRPC" != "no"; then
src/core/ext/transport/chttp2/transport/frame_settings.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_table.cc \
src/core/ext/transport/chttp2/transport/http2_settings.cc \

@ -224,6 +224,7 @@ if (PHP_GRPC != "no") {
"src\\core\\ext\\transport\\chttp2\\transport\\frame_settings.cc " +
"src\\core\\ext\\transport\\chttp2\\transport\\frame_window_update.cc " +
"src\\core\\ext\\transport\\chttp2\\transport\\hpack_encoder.cc " +
"src\\core\\ext\\transport\\chttp2\\transport\\hpack_mapping.cc " +
"src\\core\\ext\\transport\\chttp2\\transport\\hpack_parser.cc " +
"src\\core\\ext\\transport\\chttp2\\transport\\hpack_table.cc " +
"src\\core\\ext\\transport\\chttp2\\transport\\http2_settings.cc " +

@ -255,6 +255,7 @@ Pod::Spec.new do |s|
'src/core/ext/transport/chttp2/transport/frame_settings.h',
'src/core/ext/transport/chttp2/transport/frame_window_update.h',
'src/core/ext/transport/chttp2/transport/hpack_encoder.h',
'src/core/ext/transport/chttp2/transport/hpack_mapping.h',
'src/core/ext/transport/chttp2/transport/hpack_parser.h',
'src/core/ext/transport/chttp2/transport/hpack_table.h',
'src/core/ext/transport/chttp2/transport/http2_settings.h',

@ -263,6 +263,7 @@ Pod::Spec.new do |s|
'src/core/ext/transport/chttp2/transport/frame_settings.h',
'src/core/ext/transport/chttp2/transport/frame_window_update.h',
'src/core/ext/transport/chttp2/transport/hpack_encoder.h',
'src/core/ext/transport/chttp2/transport/hpack_mapping.h',
'src/core/ext/transport/chttp2/transport/hpack_parser.h',
'src/core/ext/transport/chttp2/transport/hpack_table.h',
'src/core/ext/transport/chttp2/transport/http2_settings.h',
@ -681,6 +682,7 @@ Pod::Spec.new do |s|
'src/core/ext/transport/chttp2/transport/frame_settings.cc',
'src/core/ext/transport/chttp2/transport/frame_window_update.cc',
'src/core/ext/transport/chttp2/transport/hpack_encoder.cc',
'src/core/ext/transport/chttp2/transport/hpack_mapping.cc',
'src/core/ext/transport/chttp2/transport/hpack_parser.cc',
'src/core/ext/transport/chttp2/transport/hpack_table.cc',
'src/core/ext/transport/chttp2/transport/http2_settings.cc',
@ -866,6 +868,7 @@ Pod::Spec.new do |s|
'src/core/ext/transport/chttp2/transport/frame_settings.h',
'src/core/ext/transport/chttp2/transport/frame_window_update.h',
'src/core/ext/transport/chttp2/transport/hpack_encoder.h',
'src/core/ext/transport/chttp2/transport/hpack_mapping.h',
'src/core/ext/transport/chttp2/transport/hpack_parser.h',
'src/core/ext/transport/chttp2/transport/hpack_table.h',
'src/core/ext/transport/chttp2/transport/http2_settings.h',

@ -195,6 +195,7 @@ Gem::Specification.new do |s|
s.files += %w( src/core/ext/transport/chttp2/transport/frame_settings.h )
s.files += %w( src/core/ext/transport/chttp2/transport/frame_window_update.h )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_encoder.h )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_mapping.h )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_parser.h )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_table.h )
s.files += %w( src/core/ext/transport/chttp2/transport/http2_settings.h )
@ -617,6 +618,7 @@ Gem::Specification.new do |s|
s.files += %w( src/core/ext/transport/chttp2/transport/frame_settings.cc )
s.files += %w( src/core/ext/transport/chttp2/transport/frame_window_update.cc )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_encoder.cc )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_mapping.cc )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_parser.cc )
s.files += %w( src/core/ext/transport/chttp2/transport/hpack_table.cc )
s.files += %w( src/core/ext/transport/chttp2/transport/http2_settings.cc )

@ -441,6 +441,7 @@
'src/core/ext/transport/chttp2/transport/frame_settings.cc',
'src/core/ext/transport/chttp2/transport/frame_window_update.cc',
'src/core/ext/transport/chttp2/transport/hpack_encoder.cc',
'src/core/ext/transport/chttp2/transport/hpack_mapping.cc',
'src/core/ext/transport/chttp2/transport/hpack_parser.cc',
'src/core/ext/transport/chttp2/transport/hpack_table.cc',
'src/core/ext/transport/chttp2/transport/http2_settings.cc',
@ -825,6 +826,7 @@
'src/core/ext/transport/chttp2/transport/frame_settings.cc',
'src/core/ext/transport/chttp2/transport/frame_window_update.cc',
'src/core/ext/transport/chttp2/transport/hpack_encoder.cc',
'src/core/ext/transport/chttp2/transport/hpack_mapping.cc',
'src/core/ext/transport/chttp2/transport/hpack_parser.cc',
'src/core/ext/transport/chttp2/transport/hpack_table.cc',
'src/core/ext/transport/chttp2/transport/http2_settings.cc',
@ -1060,6 +1062,7 @@
'src/core/ext/transport/chttp2/transport/frame_settings.cc',
'src/core/ext/transport/chttp2/transport/frame_window_update.cc',
'src/core/ext/transport/chttp2/transport/hpack_encoder.cc',
'src/core/ext/transport/chttp2/transport/hpack_mapping.cc',
'src/core/ext/transport/chttp2/transport/hpack_parser.cc',
'src/core/ext/transport/chttp2/transport/hpack_table.cc',
'src/core/ext/transport/chttp2/transport/http2_settings.cc',
@ -1252,6 +1255,7 @@
'src/core/ext/transport/chttp2/transport/frame_settings.cc',
'src/core/ext/transport/chttp2/transport/frame_window_update.cc',
'src/core/ext/transport/chttp2/transport/hpack_encoder.cc',
'src/core/ext/transport/chttp2/transport/hpack_mapping.cc',
'src/core/ext/transport/chttp2/transport/hpack_parser.cc',
'src/core/ext/transport/chttp2/transport/hpack_table.cc',
'src/core/ext/transport/chttp2/transport/http2_settings.cc',

@ -200,6 +200,7 @@
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_settings.h" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_window_update.h" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_encoder.h" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_mapping.h" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_parser.h" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_table.h" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/http2_settings.h" role="src" />
@ -622,6 +623,7 @@
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_settings.cc" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/frame_window_update.cc" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_encoder.cc" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_mapping.cc" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_parser.cc" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/hpack_table.cc" role="src" />
<file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/http2_settings.cc" role="src" />

@ -525,7 +525,7 @@ static void hpack_enc(grpc_chttp2_hpack_compressor* c, grpc_mdelem elem,
/* should this elem be in the table? */
size_t decoder_space_usage =
grpc_mdelem_get_size_in_hpack_table(elem, st->use_true_binary_metadata);
grpc_chttp2_get_size_in_hpack_table(elem, st->use_true_binary_metadata);
bool should_add_elem = elem_interned &&
decoder_space_usage < MAX_DECODER_SPACE_USAGE &&
c->filter_elems[HASH_FRAGMENT_1(elem_hash)] >=
@ -688,12 +688,23 @@ void grpc_chttp2_encode_header(grpc_chttp2_hpack_compressor* c,
emit_advertise_table_size_change(c, &st);
}
for (size_t i = 0; i < extra_headers_size; ++i) {
hpack_enc(c, *extra_headers[i], &st);
grpc_mdelem md = *extra_headers[i];
uint8_t static_index = grpc_chttp2_get_static_hpack_table_index(md);
if (static_index) {
emit_indexed(c, static_index, &st);
} else {
hpack_enc(c, md, &st);
}
}
grpc_metadata_batch_assert_ok(metadata);
for (grpc_linked_mdelem* l = metadata->list.head; l; l = l->next) {
uint8_t static_index = grpc_chttp2_get_static_hpack_table_index(l->md);
if (static_index) {
emit_indexed(c, static_index, &st);
} else {
hpack_enc(c, l->md, &st);
}
}
grpc_millis deadline = metadata->deadline;
if (deadline != GRPC_MILLIS_INF_FUTURE) {
deadline_enc(c, deadline, &st);

@ -0,0 +1,39 @@
/*
* Copyright 2015 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* WARNING: Auto-generated code.
*
* To make changes to this file, change
* tools/codegen/core/gen_static_metadata.py, and then re-run it.
*
* This file contains the mapping from the index of each metadata element in the
* grpc static metadata table to the index of that element in the hpack static
* metadata table. If the element is not contained in the static hpack table,
* then the returned index is 0.
*/
#include <grpc/support/port_platform.h>
#include "src/core/ext/transport/chttp2/transport/hpack_mapping.h"
const uint8_t grpc_hpack_static_mdelem_indices[GRPC_STATIC_MDELEM_COUNT] = {
0, 0, 0, 0, 0, 0, 0, 0, 3, 8, 13, 6, 7, 0, 1, 2, 0, 4,
5, 9, 10, 11, 12, 14, 15, 0, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25,
0, 0, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41,
42, 43, 44, 0, 0, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
58, 59, 60, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
};

@ -0,0 +1,38 @@
/*
* Copyright 2015 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* WARNING: Auto-generated code.
*
* To make changes to this file, change
* tools/codegen/core/gen_static_metadata.py, and then re-run it.
*
* This file contains the mapping from the index of each metadata element in the
* grpc static metadata table to the index of that element in the hpack static
* metadata table. If the element is not contained in the static hpack table,
* then the returned index is 0.
*/
#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_MAPPING_H
#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_MAPPING_H
#include <grpc/support/port_platform.h>
#include "src/core/lib/transport/static_metadata.h"
extern const uint8_t grpc_hpack_static_mdelem_indices[GRPC_STATIC_MDELEM_COUNT];
#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_MAPPING_H */

@ -27,8 +27,10 @@
#include <grpc/support/log.h>
#include <grpc/support/string_util.h>
#include "src/core/ext/transport/chttp2/transport/hpack_mapping.h"
#include "src/core/lib/debug/trace.h"
#include "src/core/lib/gpr/murmur_hash.h"
#include "src/core/lib/transport/static_metadata.h"
extern grpc_core::TraceFlag grpc_http_trace;
@ -366,3 +368,30 @@ grpc_chttp2_hptbl_find_result grpc_chttp2_hptbl_find(
return r;
}
static size_t get_base64_encoded_size(size_t raw_length) {
static const uint8_t tail_xtra[3] = {0, 2, 3};
return raw_length / 3 * 4 + tail_xtra[raw_length % 3];
}
size_t grpc_chttp2_get_size_in_hpack_table(grpc_mdelem elem,
bool use_true_binary_metadata) {
size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(GRPC_MDKEY(elem));
size_t value_len = GRPC_SLICE_LENGTH(GRPC_MDVALUE(elem));
if (grpc_is_binary_header(GRPC_MDKEY(elem))) {
return overhead_and_key + (use_true_binary_metadata
? value_len + 1
: get_base64_encoded_size(value_len));
} else {
return overhead_and_key + value_len;
}
}
uint8_t grpc_chttp2_get_static_hpack_table_index(grpc_mdelem md) {
if (GRPC_MDELEM_STORAGE(md) == GRPC_MDELEM_STORAGE_STATIC) {
return grpc_hpack_static_mdelem_indices[GRPC_MDELEM_DATA(md) -
grpc_static_mdelem_table];
} else {
return 0;
}
}

@ -83,6 +83,15 @@ grpc_mdelem grpc_chttp2_hptbl_lookup(const grpc_chttp2_hptbl* tbl,
/* add a table entry to the index */
grpc_error* grpc_chttp2_hptbl_add(grpc_chttp2_hptbl* tbl,
grpc_mdelem md) GRPC_MUST_USE_RESULT;
size_t grpc_chttp2_get_size_in_hpack_table(grpc_mdelem elem,
bool use_true_binary_metadata);
/* Returns the static hpack table index that corresponds to /a elem. Returns 0
if /a elem is not statically stored or if it is not in the static hpack
table */
uint8_t grpc_chttp2_get_static_hpack_table_index(grpc_mdelem md);
/* Find a key/value pair in the table... returns the index in the table of the
most similar entry, or 0 if the value was not found */
typedef struct {

@ -342,24 +342,6 @@ grpc_mdelem grpc_mdelem_from_grpc_metadata(grpc_metadata* metadata) {
changed ? nullptr : reinterpret_cast<grpc_mdelem_data*>(metadata));
}
static size_t get_base64_encoded_size(size_t raw_length) {
static const uint8_t tail_xtra[3] = {0, 2, 3};
return raw_length / 3 * 4 + tail_xtra[raw_length % 3];
}
size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem elem,
bool use_true_binary_metadata) {
size_t overhead_and_key = 32 + GRPC_SLICE_LENGTH(GRPC_MDKEY(elem));
size_t value_len = GRPC_SLICE_LENGTH(GRPC_MDVALUE(elem));
if (grpc_is_binary_header(GRPC_MDKEY(elem))) {
return overhead_and_key + (use_true_binary_metadata
? value_len + 1
: get_base64_encoded_size(value_len));
} else {
return overhead_and_key + value_len;
}
}
grpc_mdelem grpc_mdelem_ref(grpc_mdelem gmd DEBUG_ARGS) {
switch (GRPC_MDELEM_STORAGE(gmd)) {
case GRPC_MDELEM_STORAGE_EXTERNAL:

@ -125,9 +125,6 @@ grpc_mdelem grpc_mdelem_create(
bool grpc_mdelem_eq(grpc_mdelem a, grpc_mdelem b);
size_t grpc_mdelem_get_size_in_hpack_table(grpc_mdelem elem,
bool use_true_binary_metadata);
/* Mutator and accessor for grpc_mdelem user data. The destructor function
is used as a type tag and is checked during user_data fetch. */
void* grpc_mdelem_get_user_data(grpc_mdelem md, void (*if_destroy_func)(void*));

@ -82,6 +82,7 @@ void grpc_metadata_batch_set_value(grpc_linked_mdelem* storage,
grpc_error* grpc_metadata_batch_link_head(grpc_metadata_batch* batch,
grpc_linked_mdelem* storage)
GRPC_MUST_USE_RESULT;
/** Add \a storage to the end of \a batch. storage->md is
assumed to be valid.
\a storage is owned by the caller and must survive for the
@ -100,6 +101,7 @@ grpc_error* grpc_metadata_batch_link_tail(grpc_metadata_batch* batch,
grpc_error* grpc_metadata_batch_add_head(
grpc_metadata_batch* batch, grpc_linked_mdelem* storage,
grpc_mdelem elem_to_add) GRPC_MUST_USE_RESULT;
/** Add \a elem_to_add as the last element in \a batch, using
\a storage as backing storage for the linked list element.
\a storage is owned by the caller and must survive for the

@ -223,6 +223,7 @@ CORE_SOURCE_FILES = [
'src/core/ext/transport/chttp2/transport/frame_settings.cc',
'src/core/ext/transport/chttp2/transport/frame_window_update.cc',
'src/core/ext/transport/chttp2/transport/hpack_encoder.cc',
'src/core/ext/transport/chttp2/transport/hpack_mapping.cc',
'src/core/ext/transport/chttp2/transport/hpack_parser.cc',
'src/core/ext/transport/chttp2/transport/hpack_table.cc',
'src/core/ext/transport/chttp2/transport/http2_settings.cc',

@ -202,7 +202,7 @@ static void verify_table_size_change_match_elem_size(const char* key,
grpc_mdelem elem = grpc_mdelem_from_slices(
grpc_slice_intern(grpc_slice_from_static_string(key)),
grpc_slice_intern(grpc_slice_from_static_string(value)));
size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem, use_true_binary);
size_t elem_size = grpc_chttp2_get_size_in_hpack_table(elem, use_true_binary);
size_t initial_table_size = g_compressor.table_size;
grpc_linked_mdelem* e =
static_cast<grpc_linked_mdelem*>(gpr_malloc(sizeof(*e)));

@ -27,6 +27,7 @@
#include <grpc/support/string_util.h>
#include "src/core/ext/transport/chttp2/transport/bin_encoder.h"
#include "src/core/ext/transport/chttp2/transport/hpack_table.h"
#include "src/core/lib/gpr/string.h"
#include "src/core/lib/iomgr/exec_ctx.h"
#include "src/core/lib/slice/slice_internal.h"
@ -293,7 +294,7 @@ static void verify_ascii_header_size(const char* key, const char* value,
grpc_mdelem elem = grpc_mdelem_from_slices(
maybe_intern(grpc_slice_from_static_string(key), intern_key),
maybe_intern(grpc_slice_from_static_string(value), intern_value));
size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem, false);
size_t elem_size = grpc_chttp2_get_size_in_hpack_table(elem, false);
size_t expected_size = 32 + strlen(key) + strlen(value);
GPR_ASSERT(expected_size == elem_size);
GRPC_MDELEM_UNREF(elem);
@ -307,7 +308,7 @@ static void verify_binary_header_size(const char* key, const uint8_t* value,
maybe_intern(grpc_slice_from_static_buffer(value, value_len),
intern_value));
GPR_ASSERT(grpc_is_binary_header(GRPC_MDKEY(elem)));
size_t elem_size = grpc_mdelem_get_size_in_hpack_table(elem, false);
size_t elem_size = grpc_chttp2_get_size_in_hpack_table(elem, false);
grpc_slice value_slice = grpc_slice_from_copied_buffer(
reinterpret_cast<const char*>(value), value_len);
grpc_slice base64_encoded = grpc_chttp2_base64_encode(value_slice);

@ -23,10 +23,14 @@ import subprocess
import re
import perfection
# configuration: a list of either strings or 2-tuples of strings
# a single string represents a static grpc_mdstr
# a 2-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will
# also be created)
# Configuration: a list of either strings or 2-tuples of strings or 3-tuples of
# strings.
# A single string represents a static grpc_mdstr.
# A 2-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will
# also be created).
# A 3-tuple represents a static grpc_mdelem (and appropriate grpc_mdstrs will
# also be created), with the last value equivalent to the mdelem's static hpack
# table index as defined by RFC 7541
CONFIG = [
# metadata strings
@ -72,74 +76,74 @@ CONFIG = [
('grpc-encoding', 'deflate'),
('te', 'trailers'),
('content-type', 'application/grpc'),
(':method', 'POST'),
(':status', '200'),
(':status', '404'),
(':scheme', 'http'),
(':scheme', 'https'),
(':scheme', 'grpc'),
(':authority', ''),
(':method', 'GET'),
(':method', 'POST', 3),
(':status', '200', 8),
(':status', '404', 13),
(':scheme', 'http', 6),
(':scheme', 'https', 7),
(':scheme', 'grpc', 0),
(':authority', '', 1),
(':method', 'GET', 2),
(':method', 'PUT'),
(':path', '/'),
(':path', '/index.html'),
(':status', '204'),
(':status', '206'),
(':status', '304'),
(':status', '400'),
(':status', '500'),
('accept-charset', ''),
(':path', '/', 4),
(':path', '/index.html', 5),
(':status', '204', 9),
(':status', '206', 10),
(':status', '304', 11),
(':status', '400', 12),
(':status', '500', 14),
('accept-charset', '', 15),
('accept-encoding', ''),
('accept-encoding', 'gzip, deflate'),
('accept-language', ''),
('accept-ranges', ''),
('accept', ''),
('access-control-allow-origin', ''),
('age', ''),
('allow', ''),
('authorization', ''),
('cache-control', ''),
('content-disposition', ''),
('accept-encoding', 'gzip, deflate', 16),
('accept-language', '', 17),
('accept-ranges', '', 18),
('accept', '', 19),
('access-control-allow-origin', '', 20),
('age', '', 21),
('allow', '', 22),
('authorization', '', 23),
('cache-control', '', 24),
('content-disposition', '', 25),
('content-encoding', 'identity'),
('content-encoding', 'gzip'),
('content-encoding', ''),
('content-language', ''),
('content-length', ''),
('content-location', ''),
('content-range', ''),
('content-type', ''),
('cookie', ''),
('date', ''),
('etag', ''),
('expect', ''),
('expires', ''),
('from', ''),
('host', ''),
('if-match', ''),
('if-modified-since', ''),
('if-none-match', ''),
('if-range', ''),
('if-unmodified-since', ''),
('last-modified', ''),
('content-encoding', '', 26),
('content-language', '', 27),
('content-length', '', 28),
('content-location', '', 29),
('content-range', '', 30),
('content-type', '', 31),
('cookie', '', 32),
('date', '', 33),
('etag', '', 34),
('expect', '', 35),
('expires', '', 36),
('from', '', 37),
('host', '', 38),
('if-match', '', 39),
('if-modified-since', '', 40),
('if-none-match', '', 41),
('if-range', '', 42),
('if-unmodified-since', '', 43),
('last-modified', '', 44),
('lb-token', ''),
('lb-cost-bin', ''),
('link', ''),
('location', ''),
('max-forwards', ''),
('proxy-authenticate', ''),
('proxy-authorization', ''),
('range', ''),
('referer', ''),
('refresh', ''),
('retry-after', ''),
('server', ''),
('set-cookie', ''),
('strict-transport-security', ''),
('transfer-encoding', ''),
('user-agent', ''),
('vary', ''),
('via', ''),
('www-authenticate', ''),
('link', '', 45),
('location', '', 46),
('max-forwards', '', 47),
('proxy-authenticate', '', 48),
('proxy-authorization', '', 49),
('range', '', 50),
('referer', '', 51),
('refresh', '', 52),
('retry-after', '', 53),
('server', '', 54),
('set-cookie', '', 55),
('strict-transport-security', '', 56),
('transfer-encoding', '', 57),
('user-agent', '', 58),
('vary', '', 59),
('via', '', 60),
('www-authenticate', '', 61),
]
# Entries marked with is_default=True are ignored when counting
@ -323,6 +327,17 @@ else:
os.path.dirname(sys.argv[0]),
'../../../test/core/end2end/fuzzers/hpack.dictionary'), 'w')
HPACK_H = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../src/core/ext/transport/chttp2/transport/hpack_mapping.h'),
'w')
HPACK_C = open(
os.path.join(
os.path.dirname(sys.argv[0]),
'../../../src/core/ext/transport/chttp2/transport/hpack_mapping.cc'),
'w')
# copy-paste copyright notice from this file
with open(sys.argv[0]) as my_source:
copyright = []
@ -337,7 +352,8 @@ with open(sys.argv[0]) as my_source:
if line[0] != '#':
break
copyright.append(line)
put_banner([H, C], [line[2:].rstrip() for line in copyright])
put_banner([H, C, HPACK_H, HPACK_C],
[line[2:].rstrip() for line in copyright])
hex_bytes = [ord(c) for c in 'abcdefABCDEF0123456789']
@ -364,16 +380,44 @@ See metadata.h for an explanation of the interface here, and metadata.cc for
an explanation of what's going on.
""".splitlines())
put_banner([HPACK_H, HPACK_C], """WARNING: Auto-generated code.
To make changes to this file, change
tools/codegen/core/gen_static_metadata.py, and then re-run it.
This file contains the mapping from the index of each metadata element in the
grpc static metadata table to the index of that element in the hpack static
metadata table. If the element is not contained in the static hpack table, then
the returned index is 0.
""".splitlines())
print >> H, '#ifndef GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H'
print >> H, '#define GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H'
print >> H
print >> H, '#include <grpc/support/port_platform.h>'
print >> H
print >> H, '#include "src/core/lib/transport/metadata.h"'
print >> H
print >> C, '#include <grpc/support/port_platform.h>'
print >> C
print >> C, '#include "src/core/lib/transport/static_metadata.h"'
print >> C
print >> C, '#include "src/core/lib/slice/slice_internal.h"'
print >> C
print >> HPACK_H, ('#ifndef GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_'
'MAPPING_H')
print >> HPACK_H, ('#define GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_'
'MAPPING_H')
print >> HPACK_H
print >> HPACK_H, '#include <grpc/support/port_platform.h>'
print >> HPACK_H
print >> HPACK_H, '#include "src/core/lib/transport/static_metadata.h"'
print >> HPACK_H
print >> HPACK_C, '#include <grpc/support/port_platform.h>'
print >> HPACK_C
print >> HPACK_C, ('#include '
'"src/core/ext/transport/chttp2/transport/hpack_mapping.h"')
print >> HPACK_C
str_ofs = 0
id2strofs = {}
@ -450,10 +494,28 @@ print >> H, ('extern grpc_mdelem_data '
print >> H, ('extern uintptr_t '
'grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT];')
for i, elem in enumerate(all_elems):
print >> H, '/* "%s": "%s" */' % elem
print >> H, '/* "%s": "%s" */' % (elem[0], elem[1])
print >> H, ('#define %s (GRPC_MAKE_MDELEM(&grpc_static_mdelem_table[%d], '
'GRPC_MDELEM_STORAGE_STATIC))') % (mangle(elem).upper(), i)
print >> H
# Print out the chttp2 mapping between static mdelem index and the hpack static
# table index
print >> HPACK_H, ('extern const uint8_t grpc_hpack_static_mdelem_indices['
'GRPC_STATIC_MDELEM_COUNT];')
print >> HPACK_H
print >> HPACK_C, ('const uint8_t grpc_hpack_static_mdelem_indices['
'GRPC_STATIC_MDELEM_COUNT] = {')
indices = ''
for elem in all_elems:
index = 0
if len(elem) == 3:
index = elem[2]
indices += '%d,' % index
print >> HPACK_C, ' %s' % indices
print >> HPACK_C, '};'
print >> HPACK_C
print >> C, ('uintptr_t grpc_static_mdelem_user_data[GRPC_STATIC_MDELEM_COUNT] '
'= {')
print >> C, ' %s' % ','.join(
@ -546,8 +608,9 @@ print >> C, '}'
print >> C
print >> C, 'grpc_mdelem_data grpc_static_mdelem_table[GRPC_STATIC_MDELEM_COUNT] = {'
for a, b in all_elems:
print >> C, '{%s,%s},' % (slice_def(str_idx(a)), slice_def(str_idx(b)))
for elem in all_elems:
print >> C, '{%s,%s},' % (slice_def(str_idx(elem[0])),
slice_def(str_idx(elem[1])))
print >> C, '};'
print >> H, 'typedef enum {'
@ -599,5 +662,8 @@ print >> H, '#define GRPC_MDELEM_ACCEPT_STREAM_ENCODING_FOR_ALGORITHMS(algs) (GR
print >> H, '#endif /* GRPC_CORE_LIB_TRANSPORT_STATIC_METADATA_H */'
print >> HPACK_H, ('#endif /* GRPC_CORE_EXT_TRANSPORT_CHTTP2_TRANSPORT_HPACK_'
'MAPPING_H */')
H.close()
C.close()

@ -1013,6 +1013,8 @@ src/core/ext/transport/chttp2/transport/frame_window_update.cc \
src/core/ext/transport/chttp2/transport/frame_window_update.h \
src/core/ext/transport/chttp2/transport/hpack_encoder.cc \
src/core/ext/transport/chttp2/transport/hpack_encoder.h \
src/core/ext/transport/chttp2/transport/hpack_mapping.cc \
src/core/ext/transport/chttp2/transport/hpack_mapping.h \
src/core/ext/transport/chttp2/transport/hpack_parser.cc \
src/core/ext/transport/chttp2/transport/hpack_parser.h \
src/core/ext/transport/chttp2/transport/hpack_table.cc \

@ -10697,6 +10697,7 @@
"src/core/ext/transport/chttp2/transport/frame_settings.h",
"src/core/ext/transport/chttp2/transport/frame_window_update.h",
"src/core/ext/transport/chttp2/transport/hpack_encoder.h",
"src/core/ext/transport/chttp2/transport/hpack_mapping.h",
"src/core/ext/transport/chttp2/transport/hpack_parser.h",
"src/core/ext/transport/chttp2/transport/hpack_table.h",
"src/core/ext/transport/chttp2/transport/http2_settings.h",
@ -10734,6 +10735,8 @@
"src/core/ext/transport/chttp2/transport/frame_window_update.h",
"src/core/ext/transport/chttp2/transport/hpack_encoder.cc",
"src/core/ext/transport/chttp2/transport/hpack_encoder.h",
"src/core/ext/transport/chttp2/transport/hpack_mapping.cc",
"src/core/ext/transport/chttp2/transport/hpack_mapping.h",
"src/core/ext/transport/chttp2/transport/hpack_parser.cc",
"src/core/ext/transport/chttp2/transport/hpack_parser.h",
"src/core/ext/transport/chttp2/transport/hpack_table.cc",

Loading…
Cancel
Save