Binary header encoding.

Also fixes a rather embarrassing bug in bin_encoder.c.
	Change on 2014/12/12 by ctiller <ctiller@google.com>
-------------
Created by MOE: http://code.google.com/p/moe-java
MOE_MIGRATED_REVID=82024795
pull/1/merge
ctiller 10 years ago committed by Michael Lumish
parent abd90a6c30
commit 33023c4bac
  1. 252
      Makefile
  2. 20
      src/core/transport/chttp2/bin_encoder.c
  3. 2
      src/core/transport/chttp2/bin_encoder.h
  4. 19
      src/core/transport/chttp2/gen_hpack_tables.c
  5. 260
      src/core/transport/chttp2/hpack_parser.c
  6. 3
      src/core/transport/chttp2/hpack_parser.h
  7. 85
      src/core/transport/chttp2/stream_encoder.c
  8. 1
      src/core/transport/metadata.c
  9. 1
      test/core/end2end/gen_build_json.py
  10. 223
      test/core/end2end/tests/request_response_with_binary_metadata_and_payload.c
  11. 18
      test/core/transport/chttp2/bin_encoder_test.c

File diff suppressed because one or more lines are too long

@ -32,6 +32,9 @@
*/
#include "src/core/transport/chttp2/bin_encoder.h"
#include <string.h>
#include "src/core/transport/chttp2/huffsyms.h"
#include <grpc/support/log.h>
@ -123,7 +126,7 @@ gpr_slice grpc_chttp2_base64_encode(gpr_slice input) {
/* encode full triplets */
for (i = 0; i < input_triplets; i++) {
out[0] = alphabet[in[0] >> 2];
out[1] = alphabet[((in[0] & 0x2) << 4) | (in[1] >> 4)];
out[1] = alphabet[((in[0] & 0x3) << 4) | (in[1] >> 4)];
out[2] = alphabet[((in[1] & 0xf) << 2) | (in[2] >> 6)];
out[3] = alphabet[in[2] & 0x3f];
out += 4;
@ -136,13 +139,13 @@ gpr_slice grpc_chttp2_base64_encode(gpr_slice input) {
break;
case 1:
out[0] = alphabet[in[0] >> 2];
out[1] = alphabet[(in[0] & 0x2) << 4];
out[1] = alphabet[(in[0] & 0x3) << 4];
out += 2;
in += 1;
break;
case 2:
out[0] = alphabet[in[0] >> 2];
out[1] = alphabet[((in[0] & 0x2) << 4) | (in[1] >> 4)];
out[1] = alphabet[((in[0] & 0x3) << 4) | (in[1] >> 4)];
out[2] = alphabet[(in[1] & 0xf) << 2];
out += 3;
in += 2;
@ -238,7 +241,7 @@ gpr_slice grpc_chttp2_base64_encode_and_huffman_compress(gpr_slice input) {
/* encode full triplets */
for (i = 0; i < input_triplets; i++) {
enc_add2(&out, in[0] >> 2, ((in[0] & 0x2) << 4) | (in[1] >> 4));
enc_add2(&out, in[0] >> 2, ((in[0] & 0x3) << 4) | (in[1] >> 4));
enc_add2(&out, ((in[1] & 0xf) << 2) | (in[2] >> 6), in[2] & 0x3f);
in += 3;
}
@ -248,11 +251,11 @@ gpr_slice grpc_chttp2_base64_encode_and_huffman_compress(gpr_slice input) {
case 0:
break;
case 1:
enc_add2(&out, in[0] >> 2, (in[0] & 0x2) << 4);
enc_add2(&out, in[0] >> 2, (in[0] & 0x3) << 4);
in += 1;
break;
case 2:
enc_add2(&out, in[0] >> 2, ((in[0] & 0x2) << 4) | (in[1] >> 4));
enc_add2(&out, in[0] >> 2, ((in[0] & 0x3) << 4) | (in[1] >> 4));
enc_add1(&out, (in[1] & 0xf) << 2);
in += 2;
break;
@ -269,3 +272,8 @@ gpr_slice grpc_chttp2_base64_encode_and_huffman_compress(gpr_slice input) {
GPR_ASSERT(in == GPR_SLICE_END_PTR(input));
return output;
}
int grpc_is_binary_header(const char *key, size_t length) {
if (length < 5) return 0;
return 0 == memcmp(key + length - 4, "-bin", 4);
}

@ -51,4 +51,6 @@ gpr_slice grpc_chttp2_huffman_compress(gpr_slice input);
return y; */
gpr_slice grpc_chttp2_base64_encode_and_huffman_compress(gpr_slice input);
int grpc_is_binary_header(const char *key, size_t length);
#endif /* __GRPC_INTERNAL_TRANSPORT_CHTTP2_BIN_ENCODER_H_ */

@ -332,10 +332,29 @@ static void generate_base64_huff_encoder_table() {
printf("};\n");
}
static void generate_base64_inverse_table() {
static const char alphabet[] =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
unsigned char inverse[256];
int i;
memset(inverse, 255, sizeof(inverse));
for (i = 0; i < strlen(alphabet); i++) {
inverse[(unsigned char)alphabet[i]] = i;
}
printf("static const gpr_uint8 inverse_base64[256] = {");
for (i = 0; i < 256; i++) {
printf("%d,", inverse[i]);
}
printf("};\n");
}
int main(void) {
generate_huff_tables();
generate_first_byte_lut();
generate_base64_huff_encoder_table();
generate_base64_inverse_table();
return 0;
}

@ -37,13 +37,21 @@
#include <string.h>
#include <assert.h>
#include "src/core/support/murmur_hash.h"
#include "src/core/transport/chttp2/bin_encoder.h"
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/port_platform.h>
#include <grpc/support/string.h>
#include <grpc/support/useful.h>
typedef enum {
NOT_BINARY,
B64_BYTE0,
B64_BYTE1,
B64_BYTE2,
B64_BYTE3
} binary_state;
/* How parsing works:
The parser object keeps track of a function pointer which represents the
@ -68,8 +76,12 @@ static int parse_string_prefix(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end);
static int parse_key_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end);
static int parse_value_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end);
static int parse_value_string_with_indexed_key(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur,
const gpr_uint8 *end);
static int parse_value_string_with_literal_key(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur,
const gpr_uint8 *end);
static int parse_value0(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end);
@ -582,6 +594,27 @@ static const gpr_int16 emit_sub_tbl[249 * 16] = {
13, 22, 22, 22, 22, 256, 256, 256, 256,
};
static const gpr_uint8 inverse_base64[256] = {
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255,
255, 255, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255,
255, 64, 255, 255, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,
10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24,
25, 255, 255, 255, 255, 255, 255, 26, 27, 28, 29, 30, 31, 32, 33,
34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48,
49, 50, 51, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255,
};
/* emission helpers */
static void on_hdr(grpc_chttp2_hpack_parser *p, grpc_mdelem *md,
int add_to_table) {
@ -724,7 +757,7 @@ static int finish_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_incidx(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_value_string, finish_lithdr_incidx};
parse_value_string_with_indexed_key, finish_lithdr_incidx};
p->next_state = and_then;
p->index = (*cur) & 0x3f;
return parse_string_prefix(p, cur + 1, end);
@ -734,7 +767,8 @@ static int parse_lithdr_incidx(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_string_prefix, parse_value_string, finish_lithdr_incidx};
parse_string_prefix, parse_value_string_with_indexed_key,
finish_lithdr_incidx};
p->next_state = and_then;
p->index = 0x3f;
p->parsing.value = &p->index;
@ -745,8 +779,8 @@ static int parse_lithdr_incidx_x(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_incidx_v(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_key_string, parse_string_prefix, parse_value_string,
finish_lithdr_incidx_v};
parse_key_string, parse_string_prefix,
parse_value_string_with_literal_key, finish_lithdr_incidx_v};
p->next_state = and_then;
return parse_string_prefix(p, cur + 1, end);
}
@ -776,7 +810,7 @@ static int finish_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_notidx(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_value_string, finish_lithdr_notidx};
parse_value_string_with_indexed_key, finish_lithdr_notidx};
p->next_state = and_then;
p->index = (*cur) & 0xf;
return parse_string_prefix(p, cur + 1, end);
@ -786,7 +820,8 @@ static int parse_lithdr_notidx(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_string_prefix, parse_value_string, finish_lithdr_notidx};
parse_string_prefix, parse_value_string_with_indexed_key,
finish_lithdr_notidx};
p->next_state = and_then;
p->index = 0xf;
p->parsing.value = &p->index;
@ -797,8 +832,8 @@ static int parse_lithdr_notidx_x(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_notidx_v(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_key_string, parse_string_prefix, parse_value_string,
finish_lithdr_notidx_v};
parse_key_string, parse_string_prefix,
parse_value_string_with_literal_key, finish_lithdr_notidx_v};
p->next_state = and_then;
return parse_string_prefix(p, cur + 1, end);
}
@ -828,7 +863,7 @@ static int finish_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_value_string, finish_lithdr_nvridx};
parse_value_string_with_indexed_key, finish_lithdr_nvridx};
p->next_state = and_then;
p->index = (*cur) & 0xf;
return parse_string_prefix(p, cur + 1, end);
@ -838,7 +873,8 @@ static int parse_lithdr_nvridx(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_string_prefix, parse_value_string, finish_lithdr_nvridx};
parse_string_prefix, parse_value_string_with_indexed_key,
finish_lithdr_nvridx};
p->next_state = and_then;
p->index = 0xf;
p->parsing.value = &p->index;
@ -849,8 +885,8 @@ static int parse_lithdr_nvridx_x(grpc_chttp2_hpack_parser *p,
static int parse_lithdr_nvridx_v(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur, const gpr_uint8 *end) {
static const grpc_chttp2_hpack_parser_state and_then[] = {
parse_key_string, parse_string_prefix, parse_value_string,
finish_lithdr_nvridx_v};
parse_key_string, parse_string_prefix,
parse_value_string_with_literal_key, finish_lithdr_nvridx_v};
p->next_state = and_then;
return parse_string_prefix(p, cur + 1, end);
}
@ -1043,8 +1079,8 @@ static int parse_string_prefix(grpc_chttp2_hpack_parser *p,
}
/* append some bytes to a string */
static void append_string(grpc_chttp2_hpack_parser_string *str,
const gpr_uint8 *data, size_t length) {
static void append_bytes(grpc_chttp2_hpack_parser_string *str,
const gpr_uint8 *data, size_t length) {
if (length + str->length > str->capacity) {
str->capacity = str->length + length;
str->str = gpr_realloc(str->str, str->capacity);
@ -1053,45 +1089,156 @@ static void append_string(grpc_chttp2_hpack_parser_string *str,
str->length += length;
}
static int append_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
grpc_chttp2_hpack_parser_string *str = p->parsing.str;
gpr_uint32 bits;
gpr_uint8 decoded[3];
switch ((binary_state)p->binary) {
case NOT_BINARY:
append_bytes(str, cur, end - cur);
return 1;
b64_byte0:
case B64_BYTE0:
if (cur == end) {
p->binary = B64_BYTE0;
return 1;
}
bits = inverse_base64[*cur];
++cur;
if (bits == 255)
return 0;
else if (bits == 64)
goto b64_byte0;
p->base64_buffer = bits << 18;
/* fallthrough */
b64_byte1:
case B64_BYTE1:
if (cur == end) {
p->binary = B64_BYTE1;
return 1;
}
bits = inverse_base64[*cur];
++cur;
if (bits == 255)
return 0;
else if (bits == 64)
goto b64_byte1;
p->base64_buffer |= bits << 12;
/* fallthrough */
b64_byte2:
case B64_BYTE2:
if (cur == end) {
p->binary = B64_BYTE2;
return 1;
}
bits = inverse_base64[*cur];
++cur;
if (bits == 255)
return 0;
else if (bits == 64)
goto b64_byte2;
p->base64_buffer |= bits << 6;
/* fallthrough */
b64_byte3:
case B64_BYTE3:
if (cur == end) {
p->binary = B64_BYTE3;
return 1;
}
bits = inverse_base64[*cur];
++cur;
if (bits == 255)
return 0;
else if (bits == 64)
goto b64_byte3;
p->base64_buffer |= bits;
bits = p->base64_buffer;
decoded[0] = bits >> 16;
decoded[1] = bits >> 8;
decoded[2] = bits;
append_bytes(str, decoded, 3);
goto b64_byte0;
}
gpr_log(GPR_ERROR, "should never reach here");
abort();
return 1;
}
/* append a null terminator to a string */
static void finish_str(grpc_chttp2_hpack_parser_string *str) {
static int finish_str(grpc_chttp2_hpack_parser *p) {
gpr_uint8 terminator = 0;
append_string(str, &terminator, 1);
str->length--; /* don't actually count the null terminator */
gpr_uint8 decoded[2];
gpr_uint32 bits;
grpc_chttp2_hpack_parser_string *str = p->parsing.str;
switch ((binary_state)p->binary) {
case NOT_BINARY:
break;
case B64_BYTE0:
break;
case B64_BYTE1:
gpr_log(GPR_ERROR, "illegal base64 encoding");
return 0; /* illegal encoding */
case B64_BYTE2:
bits = p->base64_buffer;
if (bits & 0xffff) {
gpr_log(GPR_ERROR, "trailing bits in base64 encoding: 0x%04x",
bits & 0xffff);
return 0;
}
decoded[0] = bits >> 16;
append_bytes(str, decoded, 1);
break;
case B64_BYTE3:
bits = p->base64_buffer;
if (bits & 0xff) {
gpr_log(GPR_ERROR, "trailing bits in base64 encoding: 0x%02x",
bits & 0xff);
return 0;
}
decoded[0] = bits >> 16;
decoded[1] = bits >> 8;
append_bytes(str, decoded, 2);
break;
}
append_bytes(str, &terminator, 1);
p->parsing.str->length--; /* don't actually count the null terminator */
return 1;
}
/* decode a nibble from a huffman encoded stream */
static void huff_nibble(grpc_chttp2_hpack_parser *p, gpr_uint8 nibble) {
static int huff_nibble(grpc_chttp2_hpack_parser *p, gpr_uint8 nibble) {
gpr_int16 emit = emit_sub_tbl[16 * emit_tbl[p->huff_state] + nibble];
gpr_int16 next = next_sub_tbl[16 * next_tbl[p->huff_state] + nibble];
if (emit != -1) {
if (emit >= 0 && emit < 256) {
gpr_uint8 c = emit;
append_string(p->parsing.str, &c, 1);
if (!append_string(p, &c, (&c) + 1)) return 0;
} else {
assert(emit == 256);
}
}
p->huff_state = next;
return 1;
}
/* decode full bytes from a huffman encoded stream */
static void add_huff_bytes(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
static int add_huff_bytes(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
for (; cur != end; ++cur) {
huff_nibble(p, *cur >> 4);
huff_nibble(p, *cur & 0xf);
if (!huff_nibble(p, *cur >> 4) || !huff_nibble(p, *cur & 0xf)) return 0;
}
return 1;
}
/* decode some string bytes based on the current decoding mode
(huffman or not) */
static void add_str_bytes(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
static int add_str_bytes(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
if (p->huff) {
add_huff_bytes(p, cur, end);
return add_huff_bytes(p, cur, end);
} else {
append_string(p->parsing.str, cur, end - cur);
return append_string(p, cur, end);
}
}
@ -1101,11 +1248,10 @@ static int parse_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
size_t remaining = p->strlen - p->strgot;
size_t given = end - cur;
if (remaining <= given) {
add_str_bytes(p, cur, cur + remaining);
finish_str(p->parsing.str);
return parse_next(p, cur + remaining, end);
return add_str_bytes(p, cur, cur + remaining) && finish_str(p) &&
parse_next(p, cur + remaining, end);
} else {
add_str_bytes(p, cur, cur + given);
if (!add_str_bytes(p, cur, cur + given)) return 0;
p->strgot += given;
p->state = parse_string;
return 1;
@ -1114,25 +1260,63 @@ static int parse_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
/* begin parsing a string - performs setup, calls parse_string */
static int begin_parse_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end,
const gpr_uint8 *end, gpr_uint8 binary,
grpc_chttp2_hpack_parser_string *str) {
p->strgot = 0;
str->length = 0;
p->parsing.str = str;
p->huff_state = 0;
p->binary = binary;
return parse_string(p, cur, end);
}
/* parse the key string */
static int parse_key_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
return begin_parse_string(p, cur, end, &p->key);
return begin_parse_string(p, cur, end, NOT_BINARY, &p->key);
}
/* check if a key represents a binary header or not */
typedef enum { BINARY_HEADER, PLAINTEXT_HEADER, ERROR_HEADER } is_binary_header;
static is_binary_header is_binary_literal_header(grpc_chttp2_hpack_parser *p) {
return grpc_is_binary_header(p->key.str, p->key.length) ? BINARY_HEADER
: PLAINTEXT_HEADER;
}
static is_binary_header is_binary_indexed_header(grpc_chttp2_hpack_parser *p) {
grpc_mdelem *elem = grpc_chttp2_hptbl_lookup(&p->table, p->index);
if (!elem) return ERROR_HEADER;
return grpc_is_binary_header(
(const char *)GPR_SLICE_START_PTR(elem->key->slice),
GPR_SLICE_LENGTH(elem->key->slice))
? BINARY_HEADER
: PLAINTEXT_HEADER;
}
/* parse the value string */
static int parse_value_string(grpc_chttp2_hpack_parser *p, const gpr_uint8 *cur,
const gpr_uint8 *end) {
return begin_parse_string(p, cur, end, &p->value);
const gpr_uint8 *end, is_binary_header type) {
switch (type) {
case BINARY_HEADER:
return begin_parse_string(p, cur, end, B64_BYTE0, &p->value);
case PLAINTEXT_HEADER:
return begin_parse_string(p, cur, end, NOT_BINARY, &p->value);
case ERROR_HEADER:
return 0;
}
}
static int parse_value_string_with_indexed_key(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur,
const gpr_uint8 *end) {
return parse_value_string(p, cur, end, is_binary_indexed_header(p));
}
static int parse_value_string_with_literal_key(grpc_chttp2_hpack_parser *p,
const gpr_uint8 *cur,
const gpr_uint8 *end) {
return parse_value_string(p, cur, end, is_binary_literal_header(p));
}
/* PUBLIC INTERFACE */

@ -78,12 +78,15 @@ struct grpc_chttp2_hpack_parser {
gpr_uint32 strgot;
/* huffman decoding state */
gpr_uint16 huff_state;
/* is the string being decoded binary? */
gpr_uint8 binary;
/* is the current string huffman encoded? */
gpr_uint8 huff;
/* set by higher layers, used by grpc_chttp2_header_parser_parse to signal
it should append a metadata boundary at the end of frame */
gpr_uint8 is_boundary;
gpr_uint8 is_eof;
gpr_uint32 base64_buffer;
/* hpack table */
grpc_chttp2_hptbl table;

@ -38,6 +38,7 @@
#include <grpc/support/log.h>
#include <grpc/support/useful.h>
#include "src/core/transport/chttp2/bin_encoder.h"
#include "src/core/transport/chttp2/hpack_table.h"
#include "src/core/transport/chttp2/timeout_encoding.h"
#include "src/core/transport/chttp2/varint.h"
@ -269,62 +270,79 @@ static void emit_indexed(grpc_chttp2_hpack_compressor *c, gpr_uint32 index,
GRPC_CHTTP2_WRITE_VARINT(index, 1, 0x80, add_tiny_header_data(st, len), len);
}
static gpr_slice get_wire_value(grpc_mdelem *elem, gpr_uint8 *huffman_prefix) {
if (grpc_is_binary_header((const char *)GPR_SLICE_START_PTR(elem->key->slice),
GPR_SLICE_LENGTH(elem->key->slice))) {
*huffman_prefix = 0x80;
return grpc_mdstr_as_base64_encoded_and_huffman_compressed(elem->value);
}
/* TODO(ctiller): opportunistically compress non-binary headers */
*huffman_prefix = 0x00;
return elem->value->slice;
}
static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c,
gpr_uint32 key_index, grpc_mdstr *value,
gpr_uint32 key_index, grpc_mdelem *elem,
framer_state *st) {
int len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 2);
int len_val = GPR_SLICE_LENGTH(value->slice);
gpr_uint8 huffman_prefix;
gpr_slice value_slice = get_wire_value(elem, &huffman_prefix);
int len_val = GPR_SLICE_LENGTH(value_slice);
int len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1);
GRPC_CHTTP2_WRITE_VARINT(key_index, 2, 0x40,
add_tiny_header_data(st, len_pfx), len_pfx);
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, 0x00,
add_tiny_header_data(st, len_val_len), len_val_len);
add_header_data(st, gpr_slice_ref(value->slice));
add_header_data(st, gpr_slice_ref(value_slice));
}
static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c,
gpr_uint32 key_index, grpc_mdstr *value,
gpr_uint32 key_index, grpc_mdelem *elem,
framer_state *st) {
int len_pfx = GRPC_CHTTP2_VARINT_LENGTH(key_index, 4);
int len_val = GPR_SLICE_LENGTH(value->slice);
gpr_uint8 huffman_prefix;
gpr_slice value_slice = get_wire_value(elem, &huffman_prefix);
int len_val = GPR_SLICE_LENGTH(value_slice);
int len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1);
GRPC_CHTTP2_WRITE_VARINT(key_index, 4, 0x00,
add_tiny_header_data(st, len_pfx), len_pfx);
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, 0x00,
add_tiny_header_data(st, len_val_len), len_val_len);
add_header_data(st, gpr_slice_ref(value->slice));
add_header_data(st, gpr_slice_ref(value_slice));
}
static void emit_lithdr_incidx_v(grpc_chttp2_hpack_compressor *c,
grpc_mdstr *key, grpc_mdstr *value,
framer_state *st) {
int len_key = GPR_SLICE_LENGTH(key->slice);
int len_val = GPR_SLICE_LENGTH(value->slice);
grpc_mdelem *elem, framer_state *st) {
int len_key = GPR_SLICE_LENGTH(elem->key->slice);
gpr_uint8 huffman_prefix;
gpr_slice value_slice = get_wire_value(elem, &huffman_prefix);
int len_val = GPR_SLICE_LENGTH(value_slice);
int len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1);
int len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1);
*add_tiny_header_data(st, 1) = 0x40;
GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00,
add_tiny_header_data(st, len_key_len), len_key_len);
add_header_data(st, gpr_slice_ref(key->slice));
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, 0x00,
add_header_data(st, gpr_slice_ref(elem->key->slice));
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, huffman_prefix,
add_tiny_header_data(st, len_val_len), len_val_len);
add_header_data(st, gpr_slice_ref(value->slice));
add_header_data(st, gpr_slice_ref(value_slice));
}
static void emit_lithdr_noidx_v(grpc_chttp2_hpack_compressor *c,
grpc_mdstr *key, grpc_mdstr *value,
framer_state *st) {
int len_key = GPR_SLICE_LENGTH(key->slice);
int len_val = GPR_SLICE_LENGTH(value->slice);
grpc_mdelem *elem, framer_state *st) {
int len_key = GPR_SLICE_LENGTH(elem->key->slice);
gpr_uint8 huffman_prefix;
gpr_slice value_slice = get_wire_value(elem, &huffman_prefix);
int len_val = GPR_SLICE_LENGTH(value_slice);
int len_key_len = GRPC_CHTTP2_VARINT_LENGTH(len_key, 1);
int len_val_len = GRPC_CHTTP2_VARINT_LENGTH(len_val, 1);
*add_tiny_header_data(st, 1) = 0x00;
GRPC_CHTTP2_WRITE_VARINT(len_key, 1, 0x00,
add_tiny_header_data(st, len_key_len), len_key_len);
add_header_data(st, gpr_slice_ref(key->slice));
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, 0x00,
add_header_data(st, gpr_slice_ref(elem->key->slice));
GRPC_CHTTP2_WRITE_VARINT(len_val, 1, huffman_prefix,
add_tiny_header_data(st, len_val_len), len_val_len);
add_header_data(st, gpr_slice_ref(value->slice));
add_header_data(st, gpr_slice_ref(value_slice));
}
static gpr_uint32 dynidx(grpc_chttp2_hpack_compressor *c, gpr_uint32 index) {
@ -338,6 +356,7 @@ static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem,
gpr_uint32 key_hash = elem->key->hash;
gpr_uint32 elem_hash = key_hash ^ elem->value->hash;
size_t decoder_space_usage;
gpr_uint32 indices_key;
int should_add_elem;
inc_filter(HASH_FRAGMENT_1(elem_hash), &c->filter_elems_sum, c->filter_elems);
@ -371,35 +390,29 @@ static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem,
/* no hits for the elem... maybe there's a key? */
indices_key = c->indices_keys[HASH_FRAGMENT_2(key_hash)];
if (c->entries_keys[HASH_FRAGMENT_2(key_hash)] == elem->key &&
c->indices_keys[HASH_FRAGMENT_2(key_hash)] > c->tail_remote_index) {
indices_key > c->tail_remote_index) {
/* HIT: key (first cuckoo hash) */
if (should_add_elem) {
emit_lithdr_incidx(c,
dynidx(c, c->indices_keys[HASH_FRAGMENT_2(key_hash)]),
elem->value, st);
emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st);
add_elem(c, elem);
} else {
emit_lithdr_noidx(c,
dynidx(c, c->indices_keys[HASH_FRAGMENT_2(key_hash)]),
elem->value, st);
emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st);
grpc_mdelem_unref(elem);
}
return;
}
indices_key = c->indices_keys[HASH_FRAGMENT_3(key_hash)];
if (c->entries_keys[HASH_FRAGMENT_3(key_hash)] == elem->key &&
c->indices_keys[HASH_FRAGMENT_3(key_hash)] > c->tail_remote_index) {
indices_key > c->tail_remote_index) {
/* HIT: key (first cuckoo hash) */
if (should_add_elem) {
emit_lithdr_incidx(c,
dynidx(c, c->indices_keys[HASH_FRAGMENT_3(key_hash)]),
elem->value, st);
emit_lithdr_incidx(c, dynidx(c, indices_key), elem, st);
add_elem(c, elem);
} else {
emit_lithdr_noidx(c,
dynidx(c, c->indices_keys[HASH_FRAGMENT_3(key_hash)]),
elem->value, st);
emit_lithdr_noidx(c, dynidx(c, indices_key), elem, st);
grpc_mdelem_unref(elem);
}
return;
@ -408,10 +421,10 @@ static void hpack_enc(grpc_chttp2_hpack_compressor *c, grpc_mdelem *elem,
/* no elem, key in the table... fall back to literal emission */
if (should_add_elem) {
emit_lithdr_incidx_v(c, elem->key, elem->value, st);
emit_lithdr_incidx_v(c, elem, st);
add_elem(c, elem);
} else {
emit_lithdr_noidx_v(c, elem->key, elem->value, st);
emit_lithdr_noidx_v(c, elem, st);
grpc_mdelem_unref(elem);
}
}

@ -540,6 +540,7 @@ gpr_slice grpc_mdstr_as_base64_encoded_and_huffman_compressed(grpc_mdstr *gs) {
if (!s->has_base64_and_huffman_encoded) {
s->base64_and_huffman =
grpc_chttp2_base64_encode_and_huffman_compress(s->slice);
s->has_base64_and_huffman_encoded = 1;
}
slice = s->base64_and_huffman;
unlock(ctx);

@ -27,6 +27,7 @@ END2END_TESTS = [
'max_concurrent_streams',
'no_op',
'ping_pong_streaming',
'request_response_with_binary_metadata_and_payload',
'request_response_with_metadata_and_payload',
'request_response_with_payload',
'simple_delayed_request',

@ -0,0 +1,223 @@
/*
*
* Copyright 2014, Google Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are
* met:
*
* * Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following disclaimer
* in the documentation and/or other materials provided with the
* distribution.
* * Neither the name of Google Inc. nor the names of its
* contributors may be used to endorse or promote products derived from
* this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
*/
#include "test/core/end2end/end2end_tests.h"
#include <stdio.h>
#include <string.h>
#include <unistd.h>
#include <grpc/byte_buffer.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/time.h>
#include <grpc/support/useful.h>
#include "test/core/end2end/cq_verifier.h"
enum { TIMEOUT = 200000 };
static void *tag(gpr_intptr t) { return (void *)t; }
static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
const char *test_name,
grpc_channel_args *client_args,
grpc_channel_args *server_args) {
grpc_end2end_test_fixture f;
gpr_log(GPR_INFO, "%s/%s", test_name, config.name);
f = config.create_fixture(client_args, server_args);
config.init_client(&f, client_args);
config.init_server(&f, server_args);
return f;
}
static gpr_timespec n_seconds_time(int n) {
return gpr_time_add(gpr_now(), gpr_time_from_micros(GPR_US_PER_SEC * n));
}
static gpr_timespec five_seconds_time() { return n_seconds_time(5); }
static void drain_cq(grpc_completion_queue *cq) {
grpc_event *ev;
grpc_completion_type type;
do {
ev = grpc_completion_queue_next(cq, five_seconds_time());
GPR_ASSERT(ev);
type = ev->type;
grpc_event_finish(ev);
} while (type != GRPC_QUEUE_SHUTDOWN);
}
static void shutdown_server(grpc_end2end_test_fixture *f) {
if (!f->server) return;
grpc_server_shutdown(f->server);
grpc_server_destroy(f->server);
f->server = NULL;
}
static void shutdown_client(grpc_end2end_test_fixture *f) {
if (!f->client) return;
grpc_channel_destroy(f->client);
f->client = NULL;
}
static void end_test(grpc_end2end_test_fixture *f) {
shutdown_server(f);
shutdown_client(f);
grpc_completion_queue_shutdown(f->server_cq);
drain_cq(f->server_cq);
grpc_completion_queue_destroy(f->server_cq);
grpc_completion_queue_shutdown(f->client_cq);
drain_cq(f->client_cq);
grpc_completion_queue_destroy(f->client_cq);
}
/* Request/response with metadata and payload.*/
static void test_request_response_with_metadata_and_payload(
grpc_end2end_test_config config) {
grpc_call *c;
grpc_call *s;
grpc_status send_status = {GRPC_STATUS_UNIMPLEMENTED, "xyz"};
gpr_slice request_payload_slice = gpr_slice_from_copied_string("hello world");
gpr_slice response_payload_slice = gpr_slice_from_copied_string("hello you");
grpc_byte_buffer *request_payload =
grpc_byte_buffer_create(&request_payload_slice, 1);
grpc_byte_buffer *response_payload =
grpc_byte_buffer_create(&response_payload_slice, 1);
gpr_timespec deadline = five_seconds_time();
/* staggered lengths to ensure we hit various branches in base64 encode/decode
*/
grpc_metadata meta1 = {
"key1-bin", "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc", 13};
grpc_metadata meta2 = {
"key2-bin", "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d",
14};
grpc_metadata meta3 = {
"key3-bin",
"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee", 15};
grpc_metadata meta4 = {
"key4-bin",
"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", 16};
grpc_end2end_test_fixture f = begin_test(config, __FUNCTION__, NULL, NULL);
cq_verifier *v_client = cq_verifier_create(f.client_cq);
cq_verifier *v_server = cq_verifier_create(f.server_cq);
GPR_ASSERT(GRPC_CALL_OK == grpc_server_request_call(f.server, tag(100)));
/* byte buffer holds the slice, we can unref it already */
gpr_slice_unref(request_payload_slice);
gpr_slice_unref(response_payload_slice);
c = grpc_channel_create_call(f.client, "/foo", "test.google.com", deadline);
GPR_ASSERT(c);
/* add multiple metadata */
GPR_ASSERT(GRPC_CALL_OK == grpc_call_add_metadata(c, &meta1, 0));
GPR_ASSERT(GRPC_CALL_OK == grpc_call_add_metadata(c, &meta2, 0));
GPR_ASSERT(GRPC_CALL_OK ==
grpc_call_start_invoke(c, f.client_cq, tag(1), tag(2), tag(3), 0));
cq_expect_invoke_accepted(v_client, tag(1), GRPC_OP_OK);
cq_verify(v_client);
GPR_ASSERT(GRPC_CALL_OK ==
grpc_call_start_write(c, request_payload, tag(4), 0));
/* destroy byte buffer early to ensure async code keeps track of its contents
correctly */
grpc_byte_buffer_destroy(request_payload);
cq_expect_write_accepted(v_client, tag(4), GRPC_OP_OK);
cq_verify(v_client);
cq_expect_server_rpc_new(
v_server, &s, tag(100), "/foo", "test.google.com", deadline, "key1-bin",
"\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc", "key2-bin",
"\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d", NULL);
cq_verify(v_server);
grpc_call_server_accept(s, f.server_cq, tag(102));
/* add multiple metadata */
GPR_ASSERT(GRPC_CALL_OK == grpc_call_add_metadata(s, &meta3, 0));
GPR_ASSERT(GRPC_CALL_OK == grpc_call_add_metadata(s, &meta4, 0));
grpc_call_server_end_initial_metadata(s, 0);
GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_read(s, tag(5)));
cq_expect_read(v_server, tag(5), gpr_slice_from_copied_string("hello world"));
cq_verify(v_server);
GPR_ASSERT(GRPC_CALL_OK ==
grpc_call_start_write(s, response_payload, tag(6), 0));
/* destroy byte buffer early to ensure async code keeps track of its contents
correctly */
grpc_byte_buffer_destroy(response_payload);
cq_expect_write_accepted(v_server, tag(6), GRPC_OP_OK);
cq_verify(v_server);
/* fetch metadata.. */
cq_expect_client_metadata_read(
v_client, tag(2), "key3-bin",
"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee",
"key4-bin",
"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff", NULL);
cq_verify(v_client);
GPR_ASSERT(GRPC_CALL_OK == grpc_call_start_read(c, tag(7)));
cq_expect_read(v_client, tag(7), gpr_slice_from_copied_string("hello you"));
cq_verify(v_client);
GPR_ASSERT(GRPC_CALL_OK == grpc_call_writes_done(c, tag(8)));
GPR_ASSERT(GRPC_CALL_OK ==
grpc_call_start_write_status(s, send_status, tag(9)));
cq_expect_finish_accepted(v_client, tag(8), GRPC_OP_OK);
cq_expect_finished_with_status(v_client, tag(3), send_status, NULL);
cq_verify(v_client);
cq_expect_finish_accepted(v_server, tag(9), GRPC_OP_OK);
cq_expect_finished(v_server, tag(102), NULL);
cq_verify(v_server);
grpc_call_destroy(c);
grpc_call_destroy(s);
end_test(&f);
config.tear_down_data(&f);
cq_verifier_destroy(v_client);
cq_verifier_destroy(v_server);
}
void grpc_end2end_tests(grpc_end2end_test_config config) {
test_request_response_with_metadata_and_payload(config);
}

@ -32,6 +32,9 @@
*/
#include "src/core/transport/chttp2/bin_encoder.h"
#include <string.h>
#include <grpc/support/alloc.h>
#include <grpc/support/log.h>
#include <grpc/support/string.h>
@ -90,6 +93,7 @@ static void expect_combined_equiv(const char *s, size_t len, int line) {
gpr_free(t);
gpr_free(e);
gpr_free(g);
all_ok = 0;
}
gpr_slice_unref(input);
gpr_slice_unref(base64);
@ -100,6 +104,14 @@ static void expect_combined_equiv(const char *s, size_t len, int line) {
#define EXPECT_COMBINED_EQUIV(x) \
expect_combined_equiv(x, sizeof(x) - 1, __LINE__)
static void expect_binary_header(const char *hdr, int binary) {
if (grpc_is_binary_header(hdr, strlen(hdr)) != binary) {
gpr_log(GPR_ERROR, "FAILED: expected header '%s' to be %s", hdr,
binary ? "binary" : "not binary");
all_ok = 0;
}
}
int main(int argc, char **argv) {
/* Base64 test vectors from RFC 4648, with padding removed */
/* BASE64("") = "" */
@ -117,6 +129,8 @@ int main(int argc, char **argv) {
/* BASE64("foobar") = "Zm9vYmFy" */
EXPECT_SLICE_EQ("Zm9vYmFy", B64("foobar"));
EXPECT_SLICE_EQ("wMHCw8TF", B64("\xc0\xc1\xc2\xc3\xc4\xc5"));
/* Huffman encoding tests */
EXPECT_SLICE_EQ("\xf1\xe3\xc2\xe5\xf2\x3a\x6b\xa0\xab\x90\xf4\xff",
HUFF("www.example.com"));
@ -165,5 +179,9 @@ int main(int argc, char **argv) {
"\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef"
"\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff");
expect_binary_header("foo-bin", 1);
expect_binary_header("foo-bar", 0);
expect_binary_header("-bin", 0);
return all_ok ? 0 : 1;
}

Loading…
Cancel
Save